* Implemented the clone feature.
[?]
Oct 26, 2009, 5:01 PM
3XTHEUMP2ZOMPQWE3S5QWHIHCEJNEXGDPQB3JUVZFPS3RFMY455QCDependencies
- [2]
QNDXPVCI* Store the jobset's nixExprPath and nixExprInput fields in a build to - [3]
OOQ2D3KC* Refactoring: move fetchInput out of hydra_scheduler into a separate - [4]
TIOBBINA* Some renaming. - [5]
NI5BVF2V* In job inputs of type "build", allow the project and jobset names of - [6]
SHZLOM5M* eval-jobs -> hydra_eval_jobs. - [7]
IWHFLFVV* Randomly permute the order in which builds are added. This is - [8]
5QJP6JHS* Get dependencies from the database. - [9]
H7CNGK4O* Log evaluation errors etc. in the DB. - [10]
BTOXLRG3* Record the input containing the Nix expression (release.nix) in the - [11]
2T42QGZD* Register builds as GC roots so they don't get deleted. - [12]
TULPZ62Y* Perform builds in parallel. - [13]
L2E6EVE2* Merged the Build and Job tables. - [14]
LBNVQXUB* Build the /build stuff in a separate controller. - [15]
POPU2ATH* hydra_scheduler: use eval-jobs. - [16]
7ZHHVD6Q* Inputs of type "build" must now be declared explicitly. - [17]
XNCWZ7OT* Get the job priority from the meta.schedulingPriority attribute. - [18]
TLZ2SPBR - [19]
S6OISBQ3* Mark the "current" builds in a jobset, i.e. those corresponding to - [20]
CVDK3XJK* In the scheduler, don't check if we've already done a build (except - [21]
ZP3M3JWR* Be a bit less aggressive in rescheduling builds that have already - [22]
LZO3C2KI* Hack around those SQLite timeouts: just retry the transaction. - [23]
NLJJZVHO* Use ->update({...}) properly. - [24]
FDE3BJAP* Refactoring. - [25]
IXCUNELF* Don't bother with the Jobs.active column anymore. - [26]
VJHIHMEH* Store the meta.longDescription and meta.license attributes in the - [27]
M552HLIA* Support variant builds. - [28]
T4LLYESZ* Nix expression for building Hydra. - [29]
JK2QWPH6 - [30]
7YBYT2LQ - [31]
6BLUKEQ2* Caching of "path" inputs, and fake a revision number for those. - [32]
BVOPAMLS - [33]
YAPITGB3* Boolean inputs. - [34]
ECBA3GQO* Make the schema class names match the case of the SQL table names. - [35]
S5PV6IIM* Represent jobs explicitly in the DB. - [36]
AHTEIK7G* Added a maintainers field to the Builds table. - [37]
AFTXA575* $HYDRA_DATA environment variable. - [38]
X27GNHDV* Basic job info in the database. - [39]
GNIEG2GC* Disambiguate jobs by jobset name. I.e. jobs with the same name in - [40]
N22GPKYT* Put info about logs / build products in the DB. - [41]
RWIBJ5L4* Autoflush stdout. - [42]
5MNUNZWR* Store meta.maintainers. - [43]
WRIU3S5E* UI for cloning builds (not functional yet). - [44]
KOTB7BKV - [45]
A63IHCMX* Register GC roots properly. - [46]
N6E7R424
Change contents
- replacement in src/lib/Hydra/Controller/Build.pm at line 407
my ($nixExprPath, $nixExprInput) = Hydra::Controller::Jobset::nixExprPathFromParams $c;my ($nixExprPath, $nixExprInputName) = Hydra::Controller::Jobset::nixExprPathFromParams $c; - edit in src/lib/Hydra/Controller/Build.pm at line 412
my $inputInfo = {}; - replacement in src/lib/Hydra/Controller/Build.pm at line 422
fetchInput(# !!! fetchInput can take a long time, which might cause# the current HTTP request to time out. So maybe this# should be done asynchronously. But then error reporting# becomes harder.my $info = fetchInput( - edit in src/lib/Hydra/Controller/Build.pm at line 429
push @{$$inputInfo{$inputName}}, $info if defined $info; - replacement in src/lib/Hydra/Controller/Build.pm at line 434
$c->flash->{buildMsg} = "Build XXX added to the queue.";my ($jobs, $nixExprInput) = evalJobs($inputInfo, $nixExprInputName, $nixExprPath);my $job;foreach my $j (@{$jobs->{job}}) {print STDERR $j->{jobName}, "\n";if ($j->{jobName} eq $jobName) {error($c, "Nix expression returned multiple builds for job $jobName.")if $job;$job = $j;}}error($c, "Nix expression did not return a job named $jobName.") unless $job;my %currentBuilds;my $newBuild = checkBuild($c->model('DB'), $build->project, $build->jobset,$inputInfo, $nixExprInput, $job, \%currentBuilds);error($c, "This build has already been performed.") unless $newBuild;$c->flash->{buildMsg} = "Build " . $newBuild->id . " added to the queue."; - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 4
use feature 'switch';use XML::Simple; - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 7
use IPC::Run; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 11
our @EXPORT = qw(fetchInput);our @EXPORT = qw(fetchInput evalJobs checkBuild); - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 229
}sub inputsToArgs {my ($inputInfo) = @_;my @res = ();foreach my $input (keys %{$inputInfo}) {foreach my $alt (@{$inputInfo->{$input}}) {given ($alt->{type}) {when ("string") {push @res, "--argstr", $input, $alt->{value};}when ("boolean") {push @res, "--arg", $input, $alt->{value};}when (["svn", "path", "build"]) {push @res, "--arg", $input, ("{ outPath = builtins.storePath " . $alt->{storePath} . "" .(defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .(defined $alt->{version} ? "; version = \"" . $alt->{version} . "\"" : "") .";}");}}}}return @res;}sub captureStdoutStderr {my $stdin = ""; my $stdout; my $stderr;my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr);return ($res, $stdout, $stderr); - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 266[3.10399]
sub evalJobs {my ($inputInfo, $nixExprInputName, $nixExprPath) = @_;my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]or die "Cannot find the input containing the job expression.\n";die "Multiple alternatives for the input containing the Nix expression are not supported.\n"if scalar @{$inputInfo->{$nixExprInputName}} != 1;my $nixExprFullPath = $nixExprInput->{storePath} . "/" . $nixExprPath;(my $res, my $jobsXml, my $stderr) = captureStdoutStderr("hydra_eval_jobs", $nixExprFullPath, "--gc-roots-dir", getGCRootsDir,inputsToArgs($inputInfo));die "Cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;print STDERR "$stderr";my $jobs = XMLin($jobsXml,ForceArray => ['error', 'job', 'arg'],KeyAttr => [],SuppressEmpty => '')or die "cannot parse XML output";return ($jobs, $nixExprInput);}# Check whether to add the build described by $buildInfo.sub checkBuild {my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $currentBuilds) = @_;my $jobName = $buildInfo->{jobName};my $drvPath = $buildInfo->{drvPath};my $outPath = $buildInfo->{outPath};my $priority = 100;$priority = int($buildInfo->{schedulingPriority})if $buildInfo->{schedulingPriority} =~ /^\d+$/;my $build;txn_do($db, sub {# Update the last evaluation time in the database.my $job = $jobset->jobs->update_or_create({ name => $jobName, lastevaltime => time});$job->update({firstevaltime => time})unless defined $job->firstevaltime;# Don't add a build that has already been scheduled for this# job, or has been built but is still a "current" build for# this job. Note that this means that if the sources of a job# are changed from A to B and then reverted to A, three builds# will be performed (though the last one will probably use the# cached result from the first). This ensures that the builds# with the highest ID will always be the ones that we want in# the channels.# !!! Checking $outPath doesn't take meta-attributes into# account. For instance, do we want a new build to be# scheduled if the meta.maintainers field is changed?my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1});if (scalar(@previousBuilds) > 0) {print STDERR "already scheduled/built\n";$currentBuilds->{$_->id} = 1 foreach @previousBuilds;return;}# Nope, so add it.$build = $job->builds->create({ finished => 0, timestamp => time(), description => $buildInfo->{description}, longdescription => $buildInfo->{longDescription}, license => $buildInfo->{license}, homepage => $buildInfo->{homepage}, maintainers => $buildInfo->{maintainers}, nixname => $buildInfo->{nixName}, drvpath => $drvPath, outpath => $outPath, system => $buildInfo->{system}, iscurrent => 1, nixexprinput => $jobset->nixexprinput, nixexprpath => $jobset->nixexprpath});print STDERR "added to queue as build ", $build->id, "\n";$currentBuilds->{$build->id} = 1;$build->create_related('buildschedulinginfo',{ priority => $priority, busy => 0, locker => ""});my %inputs;$inputs{$jobset->nixexprinput} = $nixExprInput;foreach my $arg (@{$buildInfo->{arg}}) {$inputs{$arg->{name}} = $inputInfo->{$arg->{name}}->[$arg->{altnr}]|| die "invalid input";}foreach my $name (keys %inputs) {my $input = $inputs{$name};$build->buildinputs_builds->create({ name => $name, type => $input->{type}, uri => $input->{uri}, revision => $input->{revision}, value => $input->{value}, dependency => $input->{id}, path => $input->{storePath} || "" # !!! temporary hack, sha256hash => $input->{sha256hash}});}});return $build;}; - edit in src/script/hydra_scheduler.pl at line 5
use XML::Simple; - edit in src/script/hydra_scheduler.pl at line 8
use IPC::Run; - edit in src/script/hydra_scheduler.pl at line 14
sub captureStdoutStderr {my $stdin = ""; my $stdout; my $stderr;my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr);return ($res, $stdout, $stderr);} - edit in src/script/hydra_scheduler.pl at line 15
- edit in src/script/hydra_scheduler.pl at line 24[5.740]→[5.382:383](∅→∅),[5.382]→[5.382:383](∅→∅),[5.2878]→[5.698:699](∅→∅),[5.698]→[5.698:699](∅→∅),[5.699]→[4.0:163](∅→∅),[5.66]→[5.805:806](∅→∅),[4.163]→[5.805:806](∅→∅),[5.807]→[5.805:806](∅→∅),[5.3560]→[5.805:806](∅→∅),[5.805]→[5.805:806](∅→∅),[5.806]→[4.164:287](∅→∅),[4.287]→[5.1084:1085](∅→∅),[5.1084]→[5.1084:1085](∅→∅),[5.1085]→[5.0:24](∅→∅),[5.24]→[4.288:398](∅→∅),[4.398]→[5.224:225](∅→∅),[5.977]→[5.224:225](∅→∅),[5.224]→[5.224:225](∅→∅),[5.225]→[5.1393:1415](∅→∅),[5.1415]→[5.76:135](∅→∅),[5.135]→[4.399:450](∅→∅),[4.450]→[5.5255:5286](∅→∅),[5.5255]→[5.5255:5286](∅→∅),[5.5312]→[5.5312:5363](∅→∅)
# Check whether to add the build described by $buildInfo.sub checkBuild {my ($project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $currentBuilds) = @_;my $jobName = $buildInfo->{jobName};my $drvPath = $buildInfo->{drvPath};my $outPath = $buildInfo->{outPath};my $priority = 100;$priority = int($buildInfo->{schedulingPriority})if $buildInfo->{schedulingPriority} =~ /^\d+$/;txn_do($db, sub {# Update the last evaluation time in the database.my $job = $jobset->jobs->update_or_create({ name => $jobName, lastevaltime => time}); - edit in src/script/hydra_scheduler.pl at line 25[5.5364]→[4.451:545](∅→∅),[4.545]→[5.5466:5467](∅→∅),[5.5466]→[5.5466:5467](∅→∅),[5.5467]→[5.63:578](∅→∅),[5.578]→[5.0:66](∅→∅),[5.66]→[5.644:769](∅→∅),[5.644]→[5.644:769](∅→∅),[5.769]→[4.546:636](∅→∅),[5.161]→[5.4099:4142](∅→∅),[4.636]→[5.4099:4142](∅→∅),[5.863]→[5.4099:4142](∅→∅),[5.4099]→[5.4099:4142](∅→∅),[5.4142]→[5.864:997](∅→∅),[5.997]→[5.9666:9676](∅→∅),[5.5005]→[5.9666:9676](∅→∅),[5.9666]→[5.9666:9676](∅→∅),[5.9676]→[5.998:1007](∅→∅),[5.1007]→[5.5600:5627](∅→∅),[5.9677]→[5.5600:5627](∅→∅),[5.5627]→[4.637:679](∅→∅),[4.679]→[5.12279:12341](∅→∅),[5.5674]→[5.12279:12341](∅→∅),[5.12279]→[5.12279:12341](∅→∅),[5.12341]→[4.680:996](∅→∅),[4.996]→[5.10318:10386](∅→∅),[5.1166]→[5.10318:10386](∅→∅),[5.3007]→[5.10318:10386](∅→∅),[5.10318]→[5.10318:10386](∅→∅),[5.10386]→[4.997:1042](∅→∅),[4.1042]→[5.5006:5035](∅→∅),[5.10425]→[5.5006:5035](∅→∅),[5.5035]→[2.490:592](∅→∅),[2.592]→[5.10425:10441](∅→∅),[5.5035]→[5.10425:10441](∅→∅),[5.10425]→[5.10425:10441](∅→∅)
$job->update({firstevaltime => time})unless defined $job->firstevaltime;# Don't add a build that has already been scheduled for this# job, or has been built but is still a "current" build for# this job. Note that this means that if the sources of a job# are changed from A to B and then reverted to A, three builds# will be performed (though the last one will probably use the# cached result from the first). This ensures that the builds# with the highest ID will always be the ones that we want in# the channels.# !!! Checking $outPath doesn't take meta-attributes into# account. For instance, do we want a new build to be# scheduled if the meta.maintainers field is changed?my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1});if (scalar(@previousBuilds) > 0) {print "already scheduled/built\n";$currentBuilds->{$_->id} = 1 foreach @previousBuilds;return;}# Nope, so add it.my $build = $job->builds->create({ finished => 0, timestamp => time(), description => $buildInfo->{description}, longdescription => $buildInfo->{longDescription}, license => $buildInfo->{license}, homepage => $buildInfo->{homepage}, maintainers => $buildInfo->{maintainers}, nixname => $buildInfo->{nixName}, drvpath => $drvPath, outpath => $outPath, system => $buildInfo->{system}, iscurrent => 1, nixexprinput => $jobset->nixexprinput, nixexprpath => $jobset->nixexprpath}); - edit in src/script/hydra_scheduler.pl at line 26[5.10442]→[5.5036:5156](∅→∅),[5.5156]→[5.17990:18044](∅→∅),[5.10442]→[5.17990:18044](∅→∅),[5.18044]→[5.5722:5758](∅→∅),[5.5722]→[5.5722:5758](∅→∅),[5.262]→[5.12456:12524](∅→∅),[5.5758]→[5.12456:12524](∅→∅),[5.12456]→[5.12456:12524](∅→∅),[5.12524]→[5.67:143](∅→∅),[5.143]→[4.1043:1092](∅→∅),[4.1092]→[5.144:357](∅→∅),[5.1210]→[5.144:357](∅→∅),[5.357]→[5.5759:5807](∅→∅),[5.1302]→[5.5759:5807](∅→∅),[5.5807]→[5.358:390](∅→∅),[5.390]→[5.10662:10742](∅→∅),[5.1342]→[5.10662:10742](∅→∅),[5.5846]→[5.10662:10742](∅→∅),[5.10662]→[5.10662:10742](∅→∅),[5.10742]→[5.3852:3901](∅→∅),[5.3901]→[5.10844:10932](∅→∅),[5.10844]→[5.10844:10932](∅→∅),[5.10932]→[5.391:464](∅→∅),[5.464]→[5.2396:2449](∅→∅),[5.11007]→[5.2396:2449](∅→∅),[5.2449]→[5.11007:11037](∅→∅),[5.11007]→[5.11007:11037](∅→∅),[5.642]→[5.11037:11045](∅→∅),[5.11037]→[5.11037:11045](∅→∅),[5.5794]→[5.1176:1179](∅→∅),[5.11045]→[5.1176:1179](∅→∅),[5.1176]→[5.1176:1179](∅→∅),[5.1179]→[5.2962:2964](∅→∅)
print "added to queue as build ", $build->id, "\n";$currentBuilds->{$build->id} = 1;$build->create_related('buildschedulinginfo',{ priority => $priority, busy => 0, locker => ""});my %inputs;$inputs{$jobset->nixexprinput} = $nixExprInput;foreach my $arg (@{$buildInfo->{arg}}) {$inputs{$arg->{name}} = $inputInfo->{$arg->{name}}->[$arg->{altnr}]|| die "invalid input";}foreach my $name (keys %inputs) {my $input = $inputs{$name};$build->buildinputs_builds->create({ name => $name, type => $input->{type}, uri => $input->{uri}, revision => $input->{revision}, value => $input->{value}, dependency => $input->{id}, path => $input->{storePath} || "" # !!! temporary hack, sha256hash => $input->{sha256hash}});}});}; - edit in src/script/hydra_scheduler.pl at line 36[5.1181]→[5.1353:1416](∅→∅),[5.1416]→[5.1316:1317](∅→∅),[5.1316]→[5.1316:1317](∅→∅),[5.1317]→[5.1417:1783](∅→∅),[5.1783]→[5.1412:1462](∅→∅),[5.1462]→[5.1824:1959](∅→∅),[5.1824]→[5.1824:1959](∅→∅),[5.1959]→[5.1463:1663](∅→∅),[5.1663]→[5.2024:2094](∅→∅),[5.2024]→[5.2024:2094](∅→∅),[5.2094]→[5.2283:2297](∅→∅),[5.2656]→[5.2283:2297](∅→∅),[5.3714]→[5.2283:2297](∅→∅),[5.2283]→[5.2283:2297](∅→∅),[5.3784]→[5.2473:2489](∅→∅),[5.2473]→[5.2473:2489](∅→∅),[5.12814]→[5.2724:2725](∅→∅),[5.2724]→[5.2724:2725](∅→∅),[5.2725]→[5.2095:2112](∅→∅),[5.2112]→[5.3609:3611](∅→∅),[5.3609]→[5.3609:3611](∅→∅),[5.3611]→[5.7480:7481](∅→∅),[5.7480]→[5.7480:7481](∅→∅),[5.7481]→[5.3981:3982](∅→∅)
sub inputsToArgs {my ($inputInfo) = @_;my @res = ();foreach my $input (keys %{$inputInfo}) {foreach my $alt (@{$inputInfo->{$input}}) {given ($alt->{type}) {when ("string") {push @res, "--argstr", $input, $alt->{value};}when ("boolean") {push @res, "--arg", $input, $alt->{value};}when (["svn", "path", "build"]) {push @res, "--arg", $input, ("{ outPath = builtins.storePath " . $alt->{storePath} . "" .(defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .(defined $alt->{version} ? "; version = \"" . $alt->{version} . "\"" : "") .";}");}}}}return @res;} - replacement in src/script/hydra_scheduler.pl at line 54[5.544]→[5.544:608](∅→∅),[5.608]→[5.2302:2372](∅→∅),[5.2302]→[5.2302:2372](∅→∅),[5.2372]→[5.1443:1600](∅→∅),[5.1600]→[5.609:688](∅→∅),[5.2372]→[5.609:688](∅→∅),[5.688]→[5.3983:3984](∅→∅),[5.2420]→[5.3983:3984](∅→∅),[5.7582]→[5.3983:3984](∅→∅),[5.7583]→[5.4024:4086](∅→∅),[5.4086]→[5.1164:1273](∅→∅),[5.687]→[5.4164:4252](∅→∅),[5.1273]→[5.4164:4252](∅→∅),[5.2483]→[5.4164:4252](∅→∅),[5.4164]→[5.4164:4252](∅→∅),[5.4252]→[5.1274:1303](∅→∅)
my $nixExprInput = $inputInfo->{$jobset->nixexprinput}->[0]or die "cannot find the input containing the job expression";die "multiple alternatives for the input containing the Nix expression are not supported"if scalar @{$inputInfo->{$jobset->nixexprinput}} != 1;my $nixExprPath = $nixExprInput->{storePath} . "/" . $jobset->nixexprpath;(my $res, my $jobsXml, my $stderr) = captureStdoutStderr("hydra_eval_jobs", $nixExprPath, "--gc-roots-dir", getGCRootsDir,inputsToArgs($inputInfo));die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;print STDERR "$stderr";my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath); - edit in src/script/hydra_scheduler.pl at line 56[5.7810]→[5.7810:7841](∅→∅),[5.7841]→[5.2484:2622](∅→∅),[5.2622]→[5.8027:8069](∅→∅),[5.8027]→[5.8027:8069](∅→∅),[5.2817]→[5.8109:8110](∅→∅),[5.8109]→[5.8109:8110](∅→∅)
my $jobs = XMLin($jobsXml,ForceArray => ['error', 'job', 'arg'],KeyAttr => [],SuppressEmpty => '')or die "cannot parse XML output"; - replacement in src/script/hydra_scheduler.pl at line 61
checkBuild($project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);