Declarative projects

{"header":{"message":"declarative projects: support fully static, declarative configuration","description":null,"timestamp":"2020-09-02T16:34:34Z","authors":[{"email":"graham@grahamc.com","name":"Graham Christensen"}]},"hash":"BCFAS6VOEDLANURVQYMHK4FQY5JMBD6U2KMRZKS2T73LBP4T3QIAC","authors":[],"deps":{"hashes":[["BCFAS6VOEDLANURVQYMHK4FQY5JMBD6U2KMRZKS2T73LBP4T3QIAC","declarative projects: support fully static, declarative configuration"],["AA",null],["5PXVIXLHL7DRNGY64OQS455JEOM2CPUIJ7LEMRXVGJ2THMN4TO2AC","Declarative projects: Include the fetch info for the declarative input in the project eval"],["VDT6CUITZL2M3DSFHKAD7VYSW354AU4GDMJ7PWTFLTPUCFKTJSZAC","doc: add basic docs about declarative projects"],["CUFVKLLAL54OHDMYUHVJZK46LLX7RJMUJUE5HWYDCGIOQ5TIGXZQC","Remove Hydra::Helper::nix::txn_do from the Perl code"],["CQZQE32VR5JFHR2AQZVPXCGBHLJCLIZ3IX6NA3SWO2R44XVBSAGAC","Improve handling of Perl's block eval errors"],["6WRGCITDYP7JIBYP25QIWCHWRJWFPDP2D3TJS3WO3KUHQQJAWHMQC","Enable declarative projects."],["L54KA7IMVLHLHUED7I2K5AGSDR3EHFLMR7H332TCOTZFPV72AUWQC","handleDeclarativeJobsetBuild: handle errors from readNixFile"],["HEZQ273SDMUIQFI3YWPDNSSIOYRFT3KWPOKCDCBYATBMGPLU4F6AC","improve the error messages when invalid declarative jobsets are defined"],["OOQ2D3KCLFPYNAN253PHWLBQMB6OMO2KYQWQXLTP65SQAYZWQ5LAC","* Refactoring: move fetchInput out of hydra_scheduler into a separate"],["BMSQD2ZH3T37RNPRKKKNF5DTJEGBLY4YR2BZDKP7PXJGS3ZBBLWAC","Indentation"],["N22GPKYTOLZLBGTGDATQDVZ4R5APZEAOIA7L32X4UXBH4XNI7MWAC","* Put info about logs / build products in the DB."]],"deps":["5PXVIXLHL7DRNGY64OQS455JEOM2CPUIJ7LEMRXVGJ2THMN4TO2AC","VDT6CUITZL2M3DSFHKAD7VYSW354AU4GDMJ7PWTFLTPUCFKTJSZAC","CUFVKLLAL54OHDMYUHVJZK46LLX7RJMUJUE5HWYDCGIOQ5TIGXZQC","CQZQE32VR5JFHR2AQZVPXCGBHLJCLIZ3IX6NA3SWO2R44XVBSAGAC"],"known":["6WRGCITDYP7JIBYP25QIWCHWRJWFPDP2D3TJS3WO3KUHQQJAWHMQC","HEZQ273SDMUIQFI3YWPDNSSIOYRFT3KWPOKCDCBYATBMGPLU4F6AC","L54KA7IMVLHLHUED7I2K5AGSDR3EHFLMR7H332TCOTZFPV72AUWQC"]},"hunks":[{"Edit":{"inode":{"change":3,"pos":1},"path":"doc/manual/declarative-projects.xml","line":4,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":145}],"down_context":[{"change":3,"pos":145}],"flag":"BLOCK","start":0,"end":1247,"inode":{"change":3,"pos":1}}},"contents":[{"Add":""},{"Add":"Declarative projects"},{"Add":""},{"Add":""},{"Add":" Hydra supports declaratively configuring a project's jobsets. This"},{"Add":" configuration can be done statically, or generated by a build job."},{"Add":""},{"Add":""},{"Add":""},{"Add":" Hydra will treat the project's declarative input as a static definition"},{"Add":" if and only if the spec file contains a dictionary of dictionaries."},{"Add":" If the value of any key in the spec is not a dictionary, it will"},{"Add":" treat the spec as a generated declarative spec."},{"Add":""},{"Add":""},{"Add":"

"},{"Add":""},{"Add":" Static, Declarative Projects"},{"Add":" "},{"Add":" Hydra supports declarative projects, where jobsets are configured"},{"Add":" from a static JSON document in a repository."},{"Add":" "},{"Add":""},{"Add":" "},{"Add":" To configure a static declarative project, take the following steps:"},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" Create a Hydra-fetchable source like a Git repository or local path."},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" In that source, create a file called spec.json,"},{"Add":" and add the specification for all of the jobsets. Each key is jobset"},{"Add":" and each value is a jobset's specification. For example:"}]}},{"Replacement":{"inode":{"change":3,"pos":1},"path":"doc/manual/declarative-projects.xml","line":42,"ins":{"up_context":[{"change":3,"pos":146}],"down_context":[{"change":3,"pos":184}],"flag":"BLOCK","start":1248,"end":2555,"inode":{"change":3,"pos":1}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":146},"to":{"change":3,"start":146,"end":184},"introduced_by":3}],"inode":{"change":3,"pos":1}},"ins_contents":[{"Add":" "},{"Add":"{"},{"Add":" \"nixpkgs\": {"},{"Add":" \"enabled\": 1,"},{"Add":" \"hidden\": false,"},{"Add":" \"description\": \"Nixpkgs\","},{"Add":" \"nixexprinput\": \"nixpkgs\","},{"Add":" \"nixexprpath\": \"pkgs/top-level/release.nix\","},{"Add":" \"checkinterval\": 300,"},{"Add":" \"schedulingshares\": 100,"},{"Add":" \"enableemail\": false,"},{"Add":" \"emailoverride\": \"\","},{"Add":" \"keepnr\": 3,"},{"Add":" \"inputs\": {"},{"Add":" \"nixpkgs\": {"},{"Add":" \"type\": \"git\","},{"Add":" \"value\": \"git://github.com/NixOS/nixpkgs.git master\","},{"Add":" \"emailresponsible\": false"},{"Add":" }"},{"Add":" }"},{"Add":" },"},{"Add":" \"nixos\": {"},{"Add":" \"enabled\": 1,"},{"Add":" \"hidden\": false,"},{"Add":" \"description\": \"NixOS: Small Evaluation\","},{"Add":" \"nixexprinput\": \"nixpkgs\","},{"Add":" \"nixexprpath\": \"nixos/release-small.nix\","},{"Add":" \"checkinterval\": 300,"},{"Add":" \"schedulingshares\": 100,"},{"Add":" \"enableemail\": false,"},{"Add":" \"emailoverride\": \"\","},{"Add":" \"keepnr\": 3,"},{"Add":" \"inputs\": {"},{"Add":" \"nixpkgs\": {"},{"Add":" \"type\": \"git\","},{"Add":" \"value\": \"git://github.com/NixOS/nixpkgs.git master\","},{"Add":" \"emailresponsible\": false"},{"Add":" }"},{"Add":" }"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" "},{"Add":" Create a new project, and set the project's declarative input type,"},{"Add":" declarative input value, and declarative spec file to point to the"},{"Add":" source and JSON file you created in step 2."},{"Add":" "},{"Add":" "},{"Add":" "}],"del_contents":[{"Del":" Declarative projects\n"}]}},{"Replacement":{"inode":{"change":3,"pos":1},"path":"doc/manual/declarative-projects.xml","line":95,"ins":{"up_context":[{"change":3,"pos":193}],"down_context":[{"change":3,"pos":342}],"flag":"BLOCK","start":2556,"end":3079,"inode":{"change":3,"pos":1}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":193},"to":{"change":3,"start":193,"end":342},"introduced_by":3}],"inode":{"change":3,"pos":1}},"ins_contents":[{"Add":" Hydra will create a special jobset named .jobsets."},{"Add":" When the .jobsets jobset is evaluated, this static"},{"Add":" specification will be used for configuring the rest of the project's"},{"Add":" jobsets."},{"Add":" "},{"Add":"

"},{"Add":""},{"Add":"

"},{"Add":""},{"Add":" Generated, Declarative Projects"},{"Add":" "},{"Add":" "},{"Add":" Hydra also supports generated declarative projects, where jobsets are"},{"Add":" configured automatically from specification files instead of being"}],"del_contents":[{"Del":" Hydra also supports declarative projects, where jobsets are generated\n and configured automatically from specification files instead of being\n"}]}},{"Edit":{"inode":{"change":3,"pos":1},"path":"doc/manual/declarative-projects.xml","line":197,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":3832}],"down_context":[],"flag":"BLOCK","start":3080,"end":3091,"inode":{"change":3,"pos":1}}},"contents":[{"Add":"

"}]}},{"Edit":{"inode":{"change":10,"pos":2663},"path":"src/lib/Hydra/Helper/AddBuilds.pm","line":24,"atom":{"NewVertex":{"up_context":[{"change":6,"pos":802}],"down_context":[{"change":11,"pos":156}],"flag":"BLOCK","start":3092,"end":3124,"inode":{"change":10,"pos":2663}}},"contents":[{"Add":" handleDeclarativeJobsetJson"}]}},{"Edit":{"inode":{"change":10,"pos":2663},"path":"src/lib/Hydra/Helper/AddBuilds.pm","line":69,"atom":{"NewVertex":{"up_context":[{"change":6,"pos":1919}],"down_context":[{"change":6,"pos":1919}],"flag":"BLOCK","start":3125,"end":3803,"inode":{"change":10,"pos":2663}}},"contents":[{"Add":"sub handleDeclarativeJobsetJson {"},{"Add":" my ($db, $project, $declSpec) = @_;"},{"Add":" $db->txn_do(sub {"},{"Add":" my @kept = keys %$declSpec;"},{"Add":" push @kept, \".jobsets\";"},{"Add":" $project->jobsets->search({ name => { \"not in\" => \\@kept } })->update({ enabled => 0, hidden => 1 });"},{"Add":" while ((my $jobsetName, my $spec) = each %$declSpec) {"},{"Add":" eval {"},{"Add":" updateDeclarativeJobset($db, $project, $jobsetName, $spec);"},{"Add":" 1;"},{"Add":" } or do {"},{"Add":" print STDERR \"ERROR: failed to process declarative jobset \", $project->name, \":${jobsetName}, \", $@, \"\\n\";"},{"Add":" }"},{"Add":" }"},{"Add":" });"},{"Add":"}"}]}},{"Replacement":{"inode":{"change":10,"pos":2663},"path":"src/lib/Hydra/Helper/AddBuilds.pm","line":102,"ins":{"up_context":[{"change":6,"pos":2344}],"down_context":[{"change":5,"pos":156}],"flag":"BLOCK","start":3804,"end":3867,"inode":{"change":10,"pos":2663}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":2344},"to":{"change":4,"start":692,"end":718},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":718},"to":{"change":6,"start":2370,"end":2627},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":2370},"to":{"change":6,"start":2370,"end":2627},"introduced_by":6},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":2627},"to":{"change":6,"start":0,"end":103},"introduced_by":6},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":103},"to":{"change":5,"start":106,"end":155},"introduced_by":5},{"previous":"","flag":"DELETED","from":{"change":5,"pos":155},"to":{"change":6,"start":148,"end":293},"introduced_by":5},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":148},"to":{"change":6,"start":148,"end":293},"introduced_by":6},{"previous":"","flag":"DELETED","from":{"change":6,"pos":293},"to":{"change":6,"start":2703,"end":2729},"introduced_by":6},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":2703},"to":{"change":6,"start":2703,"end":2729},"introduced_by":6}],"inode":{"change":10,"pos":2663}},"ins_contents":[{"Add":" handleDeclarativeJobsetJson($db, $project, $declSpec);"}],"del_contents":[{"Del":" $db->txn_do(sub {\n"},{"Del":" my @kept = keys %$declSpec;\n push @kept, \".jobsets\";\n $project->jobsets->search({ name => { \"not in\" => \\@kept } })->update({ enabled => 0, hidden => 1 });\n while ((my $jobsetName, my $spec) = each %$declSpec) {\n"},{"Del":" eval {\n updateDeclarativeJobset($db, $project, $jobsetName, $spec);\n"},{"Del":" 1;\n } or do {\n"},{"Del":" print STDERR \"ERROR: failed to process declarative jobset \", $project->name, \":${jobsetName}, \", $@, \"\\n\";\n }\n"},{"Del":" }\n });\n"}]}},{"Edit":{"inode":{"change":12,"pos":1319},"path":"src/script/hydra-eval-jobset","line":572,"atom":{"NewVertex":{"up_context":[{"change":6,"pos":5233}],"down_context":[{"change":6,"pos":5233}],"flag":"BLOCK","start":3868,"end":3869,"inode":{"change":12,"pos":1319}}},"contents":[{"Add":""}]}},{"Replacement":{"inode":{"change":12,"pos":1319},"path":"src/script/hydra-eval-jobset","line":574,"ins":{"up_context":[{"change":6,"pos":5316}],"down_context":[{"change":6,"pos":5421}],"flag":"BLOCK","start":3870,"end":4786,"inode":{"change":12,"pos":1319}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":5316},"to":{"change":6,"start":5316,"end":5421},"introduced_by":6},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":6,"pos":5421},"to":{"change":2,"start":25,"end":77},"introduced_by":2}],"inode":{"change":12,"pos":1319}},"ins_contents":[{"Add":""},{"Add":" if (ref $declSpec eq \"HASH\") {"},{"Add":" if (grep ref $_ eq \"HASH\", values %$declSpec) {"},{"Add":" # Since all of its keys are hashes, assume the json document"},{"Add":" # itself is the entire set of jobs"},{"Add":" handleDeclarativeJobsetJson($db, $project, $declSpec);"},{"Add":" $db->txn_do(sub {"},{"Add":" $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });"},{"Add":" });"},{"Add":" return;"},{"Add":" } else {"},{"Add":" # Update the jobset with the spec's inputs, and the continue"},{"Add":" # evaluating the .jobsets jobset."},{"Add":" updateDeclarativeJobset($db, $project, \".jobsets\", $declSpec);"},{"Add":" $jobset->discard_changes;"},{"Add":" $inputInfo->{\"declInput\"} = [ $declInput ];"},{"Add":" }"},{"Add":" } else {"},{"Add":" die \"Declarative specification file $declFile is not a dictionary\""},{"Add":" }"}],"del_contents":[{"Del":" updateDeclarativeJobset($db, $project, \".jobsets\", $declSpec);\n $jobset->discard_changes;\n"},{"Del":" $inputInfo->{\"declInput\"} = [ $declInput ];\n"}]}}],"owner":"grahamc","repo":"hydra2","token":"SB7oU7AJ0nvRiXYNkX/w391wZqX+h75Q/VFtfsyXS0c=","login":null}

…

Dependencies

Change contents