[?]
Nov 10, 2008, 1:33 PM
7YBYT2LQML2PKEO6UO4444AGSASS664UCDXW2YO3ALB7THQHCEBQCDependencies
- [2]
BVOPAMLS - [3]
X27GNHDV* Basic job info in the database. - [4]
PHX2HIVG* Store info about the build inputs in the build record. - [5]
VCOSLZRP - [6]
5QJP6JHS* Get dependencies from the database. - [7]
N22GPKYT* Put info about logs / build products in the DB. - [8]
M552HLIA* Support variant builds. - [9]
FDE3BJAP* Refactoring. - [10]
TWURROKC - [11]
DVNWJXWW* Generic declaration of build products. - [12]
WYN733ST* Store build duration, handle cached builds. - [13]
67P45PY4 - [14]
GWCV3TQV* BuildInputs table: link to dependencies, include store paths. - [15]
J5UVLXOK* Start of a basic Catalyst web interface. - [16]
ELCI5T2A* Show the latest build for each job. - [17]
T7AHGVGM - [18]
ZEHSSVFG - [*]
UVMFS73T* Some jQuery / CSS hackery. - [*]
CLXEECMF* Start putting build results in a database.
Change contents
- replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm at line 24
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pt0CJFX1pP9Z2TjqrTjTkw# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eMNna7u2l0ec+OYuvtGRpg - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm at line 24
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3NKUaF4u4H6ZmIRCeva8yA# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LaXQ4zxxvzdKFBRVcjMdMQ - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm at line 68
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8s5Z03ugocOVb021EwGVag# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C1XPkCXQImyXduKER0Dllg - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Inputs.pm at line 38
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:AzV6B/6CCrroPlO32n2p3A# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:A3Is4VTFkTl2DzrYjzdrZA - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 49
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GubRofAmJ/sbJbjyV3aKSQ# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZF8UB1MtbPuOk7wTSFJR5Q - edit in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 52
__PACKAGE__->has_many("inputs","HydraFrontend::Schema::Inputs",{ "foreign.job" => "self.id" },); - edit in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 58
# You can replace this text with custom content, and it will be preserved on regeneration - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputalts.pm at line 36
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZjjWLbAWExxOqsDz41A3KA# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ibTncC1AslPWt1eiTtwplA - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm at line 46
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6hzbFjPWQ872UxFhhpxjFg# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:D1UzSZwPtwDmOI7q6g8uKQ - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm at line 59
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oRV4yw0DWG5PI0agcM7QHA# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6Pyrgervmq03S5Nx8QfA1Q - replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm at line 32
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9SeEXSEOH1ocrdkoa7fx5Q# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p8LbF31qRl/JfMK5wfkeCg - replacement in src/HydraFrontend/lib/HydraFrontend/Schema.pm at line 11
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xP97YDrN7Bm2B/BlbQJ7fQ# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dBO/r6lVlITiJ/HlltKcpQ - edit in src/HydraFrontend/root/hydra.css at line 160
tr.runningJob {background-color: #ff3030;color: white;} - edit in src/HydraFrontend/root/hydra.css at line 165
- replacement in src/HydraFrontend/root/index.tt at line 12
<tr><tr [% IF job.busy %]class="runningJob"[% END %] > - file addition: project.tt[3.7332]
[% WRAPPER layout.tt title="Hydra Overview" %]<h1>Project <tt>[% project.name %]</tt></h1><h2>Definition</h2>[% FOREACH jobset IN project.jobsets -%]<h3>Jobset <tt>[% jobset.name %]</tt></h3><p>Description: [% jobset.description %]<br />Nix expression: <tt>[% jobset.nixexprpath %]</tt> in input <tt>[% jobset.nixexprinput %]</tt></p><table class="tablesorter"><thead><tr><th>Input name</th><th>Type</th><th>Values</th></tr></thead><tbody>[% FOREACH input IN jobset.jobsetinputs -%]<tr><td><tt>[% input.name %]</tt></td><td><tt>[% input.type %]</tt></td><td>[% FOREACH alt IN input.jobsetinputalts -%][% IF input.type == "string" %]<tt>"[% alt.value %]"</tt>[% ELSE %]<tt>[% alt.uri %]</tt>[% END %][% END %]</td></tr>[% END %]</tbody></table>[% END -%]<h2>Jobs</h2><ul>[% FOREACH jobName IN jobNames -%]<li><a href="[% c.uri_for('/job' project.name jobName.attrname) %]"><tt>[% jobName.attrname %]</tt></a></li>[% END %]</ul>[% END %] - file addition: build.pl[21.4]
#! @perl@ -wuse strict;use File::Basename;use HydraFrontend::Schema;my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});sub isValidPath {my $path = shift;return system("nix-store --check-validity $path 2> /dev/null") == 0;}sub buildJob {my ($job) = @_;my $drvPath = $job->drvpath;my $outPath = $job->outpath;my $isCachedBuild = 1;my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)my $startTime = 0;my $stopTime = 0;if (!isValidPath($outPath)) {$isCachedBuild = 0;$startTime = time();print " BUILDING\n";my $res = system("nix-store --realise $drvPath");$stopTime = time();$outputCreated = $res == 0;}my $buildStatus;if ($outputCreated) {# "Positive" failures, e.g. the builder returned exit code 0# but flagged some error condition.$buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;} else {$buildStatus = 1; # = Nix failure}$db->txn_do(sub {my $build = $db->resultset('Builds')->create({ timestamp => time(), project => $job->project->name, jobset => $job->jobset->name, attrname => $job->attrname, description => $job->description, drvpath => $drvPath, outpath => $outPath, iscachedbuild => $isCachedBuild, buildstatus => $buildStatus, starttime => $startTime, stoptime => $stopTime, system => $job->system});print " build ID = ", $build->id, "\n";foreach my $input ($job->inputs) {$input->job(undef);$input->build($build->id);$input->update;}my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;if (-e $logPath) {print " LOG $logPath\n";$db->resultset('Buildlogs')->create({ build => $build->id, logphase => "full", path => $logPath, type => "raw"});}if ($outputCreated) {if (-e "$outPath/log") {foreach my $logPath (glob "$outPath/log/*") {print " LOG $logPath\n";$db->resultset('Buildlogs')->create({ build => $build->id, logphase => basename($logPath), path => $logPath, type => "raw"});}}if (-e "$outPath/nix-support/hydra-build-products") {open LIST, "$outPath/nix-support/hydra-build-products" or die;while (<LIST>) {/^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;my $type = $1;my $subtype = $2;my $path = $3;die unless -e $path;$db->resultset('Buildproducts')->create({ build => $build->id, type => $type, subtype => $subtype, path => $path});}close LIST;} else {$db->resultset('Buildproducts')->create({ build => $build->id, type => "nix-build", subtype => "", path => $outPath});}}$job->delete;});}my $jobId = $ARGV[0] or die;print "building job $jobId\n";# Lock the job. If necessary, steal the lock from the parent process# (runner.pl). This is so that if the runner dies, the children# (i.e. the job builders) can continue to run and won't have the lock# taken away.my $job;$db->txn_do(sub {($job) = $db->resultset('Jobs')->search({ id => $jobId });die "job $jobId doesn't exist" unless defined $job;if ($job->busy != 0 && $job->locker != getppid) {die "job $jobId is already being built";}$job->busy(1);$job->locker($$);$job->update;});die unless $job;# Build the job. If it throws an error, unlock the job so that it can# be retried.eval {print "BUILD\n";buildJob $job;print "DONE\n";};if ($@) {warn $@;$db->txn_do(sub {$job->busy(0);$job->locker($$);$job->update;});} - replacement in src/hydra.sql at line 132
priority integer not null,priority integer not null default 0, - replacement in src/hydra.sql at line 134
busy integer not null, -- true means someone is building this job nowlocker text not null, -- !!! hostname/pid of the process building this job?busy integer not null default 0, -- true means someone is building this job nowlocker text not null default '', -- !!! hostname/pid of the process building this job? - file addition: runner.pl[21.4]
#! @perl@ -wuse strict;use HydraFrontend::Schema;my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});# Unlock jobs whose building process has died.$db->txn_do(sub {my @jobs = $db->resultset('Jobs')->search({ busy => 1 });foreach my $job (@jobs) {my $pid = $job->locker;if (kill(0, $pid) != 1) { # see if we can signal the processprint "job ", $job->id, " pid $pid died, unlocking\n";$job->busy(0);$job->locker("");$job->update;}}});while (1) {print "looking for runnable jobs...\n";my $job;$db->txn_do(sub {my @jobs = $db->resultset('Jobs')->search({ busy => 0 }, {order_by => ["priority", "timestamp"]});print "# of available jobs: ", scalar(@jobs), "\n";if (scalar @jobs > 0) {$job = $jobs[0];$job->busy(1);$job->locker($$);$job->update;}});# Start the job. We need to do this outside the transaction in# case it aborts or something.if (defined $job) {print "starting job ", $job->id, "\n";eval {system("perl -I HydraFrontend/lib -w ./build.pl " . $job->id);};if ($@) {warn $@;$db->txn_do(sub {$job->busy(0);$job->locker($$);$job->update;});}}print "sleeping...\n";sleep(10);} - edit in src/scheduler.pl at line 5
use File::Basename; - edit in src/scheduler.pl at line 14[3.113]→[3.113:117](∅→∅),[3.117]→[3.5762:5777](∅→∅),[3.1426]→[3.5762:5777](∅→∅),[3.5777]→[2.5396:5495](∅→∅),[3.3059]→[3.1492:1493](∅→∅),[3.4751]→[3.1492:1493](∅→∅),[2.5495]→[3.1492:1493](∅→∅),[3.5840]→[3.1492:1493](∅→∅),[3.1492]→[3.1492:1493](∅→∅),[3.1493]→[3.5841:6048](∅→∅),[3.6048]→[3.2311:2317](∅→∅),[3.2311]→[3.2311:2317](∅→∅),[3.2317]→[3.2305:2333](∅→∅),[3.2333]→[3.0:96](∅→∅),[3.96]→[3.2358:2408](∅→∅),[3.2358]→[3.2358:2408](∅→∅),[3.2408]→[3.118:152](∅→∅),[3.152]→[3.2483:2511](∅→∅),[3.2483]→[3.2483:2511](∅→∅),[3.2511]→[3.2317:2318](∅→∅),[3.2317]→[3.2317:2318](∅→∅),[3.2318]→[3.2512:2542](∅→∅),[3.2542]→[3.6049:6142](∅→∅),[3.6142]→[3.2607:2636](∅→∅),[3.2607]→[3.2607:2636](∅→∅),[3.2636]→[3.2379:2380](∅→∅),[3.2379]→[3.2379:2380](∅→∅),[3.2380]→[3.97:425](∅→∅),[3.425]→[3.2679:2685](∅→∅),[3.2679]→[3.2679:2685](∅→∅),[3.2685]→[3.2421:2422](∅→∅),[3.2421]→[3.2421:2422](∅→∅),[3.2422]→[3.6143:6366](∅→∅),[3.6366]→[2.5496:5538](∅→∅),[2.5538]→[3.6366:6596](∅→∅),[3.6366]→[3.6366:6596](∅→∅),[3.6596]→[3.4752:4784](∅→∅),[3.4784]→[3.6596:6612](∅→∅),[3.6596]→[3.6596:6612](∅→∅),[3.6612]→[3.0:53](∅→∅),[3.53]→[3.6620:6621](∅→∅),[3.6620]→[3.6620:6621](∅→∅),[3.6621]→[3.2002:2107](∅→∅),[3.2107]→[3.5794:5878](∅→∅),[3.5878]→[3.4876:5138](∅→∅),[3.4876]→[3.4876:5138](∅→∅),[3.5138]→[3.5879:5924](∅→∅),[3.5924]→[3.5180:5275](∅→∅),[3.5180]→[3.5180:5275](∅→∅),[3.2904]→[3.3535:3546](∅→∅),[3.5275]→[3.3535:3546](∅→∅),[3.3535]→[3.3535:3546](∅→∅),[3.3546]→[3.54:240](∅→∅),[3.6621]→[3.54:240](∅→∅),[3.240]→[3.5925:5963](∅→∅),[3.5963]→[3.280:414](∅→∅),[3.280]→[3.280:414](∅→∅),[3.414]→[3.6635:6636](∅→∅),[3.6635]→[3.6635:6636](∅→∅),[3.6636]→[3.426:456](∅→∅),[3.448]→[3.6679:6680](∅→∅),[3.456]→[3.6679:6680](∅→∅),[3.4701]→[3.6679:6680](∅→∅),[3.6679]→[3.6679:6680](∅→∅),[3.680]→[3.680:886](∅→∅),[3.886]→[3.5964:6010](∅→∅),[3.6010]→[3.934:1074](∅→∅),[3.934]→[3.934:1074](∅→∅),[3.1074]→[3.4702:5208](∅→∅),[3.5208]→[3.6011:6057](∅→∅),[3.6057]→[3.5308:5434](∅→∅),[3.5308]→[3.5308:5434](∅→∅),[3.5434]→[3.1074:1120](∅→∅),[3.1074]→[3.1074:1120](∅→∅),[3.1120]→[3.5435:5541](∅→∅),[3.5541]→[3.6058:6100](∅→∅),[3.6100]→[3.5633:5774](∅→∅),[3.5633]→[3.5633:5774](∅→∅),[3.1120]→[3.3586:3610](∅→∅),[3.5774]→[3.3586:3610](∅→∅),[3.3586]→[3.3586:3610](∅→∅),[3.3610]→[3.1121:1138](∅→∅),[3.1138]→[3.3616:3617](∅→∅),[3.3616]→[3.3616:3617](∅→∅)
}sub buildJob {my ($project, $jobset, $jobName, $description, $drvPath, $outPath, $usedInputs, $system) = @_;if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {print " already done\n";return;}my $isCachedBuild = 1;my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)my $startTime = 0;my $stopTime = 0;if (!isValidPath($outPath)) {$isCachedBuild = 0;$startTime = time();print " BUILDING\n";my $res = system("nix-store --realise $drvPath");$stopTime = time();$outputCreated = $res == 0;}my $buildStatus;if ($outputCreated) {# "Positive" failures, e.g. the builder returned exit code 0# but flagged some error condition.$buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;} else {$buildStatus = 1; # = Nix failure}$db->txn_do(sub {my $build = $db->resultset('Builds')->create({ timestamp => time(), project => $project->name, jobset => $jobset->name, attrname => $jobName, description => $description, drvpath => $drvPath, outpath => $outPath, iscachedbuild => $isCachedBuild, buildstatus => $buildStatus, starttime => $startTime, stoptime => $stopTime, system => $system});print " build ID = ", $build->id, "\n";foreach my $inputName (keys %{$usedInputs}) {my $input = $usedInputs->{$inputName};$db->resultset('Inputs')->create({ build => $build->id, name => $inputName, type => $input->{type}, uri => $input->{uri}#, revision => $input->{orig}->revision#, tag => $input->{orig}->tag, value => $input->{value}, dependency => $input->{id}, path => ($input->{storePath} or "") # !!! temporary hack});}my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;if (-e $logPath) {print " LOG $logPath\n";$db->resultset('Buildlogs')->create({ build => $build->id, logphase => "full", path => $logPath, type => "raw"});}if ($outputCreated) {if (-e "$outPath/log") {foreach my $logPath (glob "$outPath/log/*") {print " LOG $logPath\n";$db->resultset('Buildlogs')->create({ build => $build->id, logphase => basename($logPath), path => $logPath, type => "raw"});}}if (-e "$outPath/nix-support/hydra-build-products") {open LIST, "$outPath/nix-support/hydra-build-products" or die;while (<LIST>) {/^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;my $type = $1;my $subtype = $2;my $path = $3;die unless -e $path;$db->resultset('Buildproducts')->create({ build => $build->id, type => $type, subtype => $subtype, path => $path});}close LIST;} else {$db->resultset('Buildproducts')->create({ build => $build->id, type => "nix-build", subtype => "", path => $outPath});}}}); - replacement in src/scheduler.pl at line 63
buildJob($project, $jobset, $jobName, $description, $drvPath, $outPath, $inputInfo, $job->{system});$db->txn_do(sub {if (scalar($db->resultset('Builds')->search({ project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath })) > 0){print " already done\n";return;}if (scalar($db->resultset('Jobs')->search({ project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath })) > 0){print " already queued\n";return;}print " adding to queue\n";my $job = $db->resultset('Jobs')->create({ timestamp => time(), priority => 0, busy => 0, locker => "", project => $project->name, jobset => $jobset->name, attrname => $jobName, description => $description, drvpath => $drvPath, outpath => $outPath, system => $job->{system}});foreach my $inputName (keys %{$inputInfo}) {my $input = $inputInfo->{$inputName};$db->resultset('Inputs')->create({ job => $job->id, name => $inputName, type => $input->{type}, uri => $input->{uri}#, revision => $input->{orig}->revision#, tag => $input->{orig}->tag, value => $input->{value}, dependency => $input->{id}, path => ($input->{storePath} or "") # !!! temporary hack});}});