[?]
Nov 10, 2008, 1:33 PM
7YBYT2LQML2PKEO6UO4444AGSASS664UCDXW2YO3ALB7THQHCEBQC

Dependencies

  • [2] BVOPAMLS
  • [3] X27GNHDV * Basic job info in the database.
  • [4] PHX2HIVG * Store info about the build inputs in the build record.
  • [5] VCOSLZRP
  • [6] 5QJP6JHS * Get dependencies from the database.
  • [7] N22GPKYT * Put info about logs / build products in the DB.
  • [8] M552HLIA * Support variant builds.
  • [9] FDE3BJAP * Refactoring.
  • [10] TWURROKC
  • [11] DVNWJXWW * Generic declaration of build products.
  • [12] WYN733ST * Store build duration, handle cached builds.
  • [13] 67P45PY4
  • [14] GWCV3TQV * BuildInputs table: link to dependencies, include store paths.
  • [15] J5UVLXOK * Start of a basic Catalyst web interface.
  • [16] ELCI5T2A * Show the latest build for each job.
  • [17] T7AHGVGM
  • [18] ZEHSSVFG
  • [*] UVMFS73T * Some jQuery / CSS hackery.
  • [*] CLXEECMF * Start putting build results in a database.

Change contents

  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm at line 24
    [3.3992][2.55:197]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pt0CJFX1pP9Z2TjqrTjTkw
    [3.3992]
    [3.4134]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eMNna7u2l0ec+OYuvtGRpg
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm at line 24
    [3.4922][2.198:340]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3NKUaF4u4H6ZmIRCeva8yA
    [3.4922]
    [3.5064]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LaXQ4zxxvzdKFBRVcjMdMQ
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm at line 68
    [3.6194][2.567:709]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8s5Z03ugocOVb021EwGVag
    [3.6194]
    [3.0]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C1XPkCXQImyXduKER0Dllg
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Inputs.pm at line 38
    [3.782][2.710:852]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:AzV6B/6CCrroPlO32n2p3A
    [3.782]
    [3.924]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:A3Is4VTFkTl2DzrYjzdrZA
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 49
    [3.2706][2.1226:1368]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GubRofAmJ/sbJbjyV3aKSQ
    [3.2706]
    [3.2848]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZF8UB1MtbPuOk7wTSFJR5Q
  • edit in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 52
    [3.2849]
    [3.2849]
    __PACKAGE__->has_many(
    "inputs",
    "HydraFrontend::Schema::Inputs",
    { "foreign.job" => "self.id" },
    );
  • edit in src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm at line 58
    [3.2850][3.2850:2940]()
    # You can replace this text with custom content, and it will be preserved on regeneration
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputalts.pm at line 36
    [3.1831][2.1409:1551]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZjjWLbAWExxOqsDz41A3KA
    [3.1831]
    [3.1973]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ibTncC1AslPWt1eiTtwplA
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm at line 46
    [3.1681][2.1589:1731]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6hzbFjPWQ872UxFhhpxjFg
    [3.1681]
    [3.1823]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:D1UzSZwPtwDmOI7q6g8uKQ
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm at line 59
    [3.2992][2.2084:2226]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oRV4yw0DWG5PI0agcM7QHA
    [3.2992]
    [3.3134]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6Pyrgervmq03S5Nx8QfA1Q
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm at line 32
    [3.3685][2.2451:2593]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9SeEXSEOH1ocrdkoa7fx5Q
    [3.3685]
    [3.3827]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p8LbF31qRl/JfMK5wfkeCg
  • replacement in src/HydraFrontend/lib/HydraFrontend/Schema.pm at line 11
    [3.6498][2.2594:2736]()
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xP97YDrN7Bm2B/BlbQJ7fQ
    [3.6498]
    [3.6640]
    # Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
    # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dBO/r6lVlITiJ/HlltKcpQ
  • edit in src/HydraFrontend/root/hydra.css at line 160
    [20.251]
    [3.2036]
    tr.runningJob {
    background-color: #ff3030;
    color: white;
    }
  • edit in src/HydraFrontend/root/hydra.css at line 165
    [3.2037]
    [20.251]
  • replacement in src/HydraFrontend/root/index.tt at line 12
    [2.3568][2.3568:3579]()
    <tr>
    [2.3568]
    [2.3579]
    <tr [% IF job.busy %]class="runningJob"[% END %] >
  • file addition: project.tt (----------)
    [3.7332]
    [% WRAPPER layout.tt title="Hydra Overview" %]
    <h1>Project <tt>[% project.name %]</tt></h1>
    <h2>Definition</h2>
    [% FOREACH jobset IN project.jobsets -%]
    <h3>Jobset <tt>[% jobset.name %]</tt></h3>
    <p>
    Description: [% jobset.description %]
    <br />
    Nix expression: <tt>[% jobset.nixexprpath %]</tt> in input <tt>[% jobset.nixexprinput %]</tt>
    </p>
    <table class="tablesorter">
    <thead>
    <tr><th>Input name</th><th>Type</th><th>Values</th></tr>
    </thead>
    <tbody>
    [% FOREACH input IN jobset.jobsetinputs -%]
    <tr>
    <td><tt>[% input.name %]</tt></td>
    <td><tt>[% input.type %]</tt></td>
    <td>
    [% FOREACH alt IN input.jobsetinputalts -%]
    [% IF input.type == "string" %]
    <tt>"[% alt.value %]"</tt>
    [% ELSE %]
    <tt>[% alt.uri %]</tt>
    [% END %]
    [% END %]
    </td>
    </tr>
    [% END %]
    </tbody>
    </table>
    [% END -%]
    <h2>Jobs</h2>
    <ul>
    [% FOREACH jobName IN jobNames -%]
    <li><a href="[% c.uri_for('/job' project.name jobName.attrname) %]"><tt>[% jobName.attrname %]</tt></a></li>
    [% END %]
    </ul>
    [% END %]
  • file addition: build.pl (----------)
    [21.4]
    #! @perl@ -w
    use strict;
    use File::Basename;
    use HydraFrontend::Schema;
    my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
    sub isValidPath {
    my $path = shift;
    return system("nix-store --check-validity $path 2> /dev/null") == 0;
    }
    sub buildJob {
    my ($job) = @_;
    my $drvPath = $job->drvpath;
    my $outPath = $job->outpath;
    my $isCachedBuild = 1;
    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
    my $startTime = 0;
    my $stopTime = 0;
    if (!isValidPath($outPath)) {
    $isCachedBuild = 0;
    $startTime = time();
    print " BUILDING\n";
    my $res = system("nix-store --realise $drvPath");
    $stopTime = time();
    $outputCreated = $res == 0;
    }
    my $buildStatus;
    if ($outputCreated) {
    # "Positive" failures, e.g. the builder returned exit code 0
    # but flagged some error condition.
    $buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
    } else {
    $buildStatus = 1; # = Nix failure
    }
    $db->txn_do(sub {
    my $build = $db->resultset('Builds')->create(
    { timestamp => time()
    , project => $job->project->name
    , jobset => $job->jobset->name
    , attrname => $job->attrname
    , description => $job->description
    , drvpath => $drvPath
    , outpath => $outPath
    , iscachedbuild => $isCachedBuild
    , buildstatus => $buildStatus
    , starttime => $startTime
    , stoptime => $stopTime
    , system => $job->system
    });
    print " build ID = ", $build->id, "\n";
    foreach my $input ($job->inputs) {
    $input->job(undef);
    $input->build($build->id);
    $input->update;
    }
    my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
    if (-e $logPath) {
    print " LOG $logPath\n";
    $db->resultset('Buildlogs')->create(
    { build => $build->id
    , logphase => "full"
    , path => $logPath
    , type => "raw"
    });
    }
    if ($outputCreated) {
    if (-e "$outPath/log") {
    foreach my $logPath (glob "$outPath/log/*") {
    print " LOG $logPath\n";
    $db->resultset('Buildlogs')->create(
    { build => $build->id
    , logphase => basename($logPath)
    , path => $logPath
    , type => "raw"
    });
    }
    }
    if (-e "$outPath/nix-support/hydra-build-products") {
    open LIST, "$outPath/nix-support/hydra-build-products" or die;
    while (<LIST>) {
    /^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
    my $type = $1;
    my $subtype = $2;
    my $path = $3;
    die unless -e $path;
    $db->resultset('Buildproducts')->create(
    { build => $build->id
    , type => $type
    , subtype => $subtype
    , path => $path
    });
    }
    close LIST;
    } else {
    $db->resultset('Buildproducts')->create(
    { build => $build->id
    , type => "nix-build"
    , subtype => ""
    , path => $outPath
    });
    }
    }
    $job->delete;
    });
    }
    my $jobId = $ARGV[0] or die;
    print "building job $jobId\n";
    # Lock the job. If necessary, steal the lock from the parent process
    # (runner.pl). This is so that if the runner dies, the children
    # (i.e. the job builders) can continue to run and won't have the lock
    # taken away.
    my $job;
    $db->txn_do(sub {
    ($job) = $db->resultset('Jobs')->search({ id => $jobId });
    die "job $jobId doesn't exist" unless defined $job;
    if ($job->busy != 0 && $job->locker != getppid) {
    die "job $jobId is already being built";
    }
    $job->busy(1);
    $job->locker($$);
    $job->update;
    });
    die unless $job;
    # Build the job. If it throws an error, unlock the job so that it can
    # be retried.
    eval {
    print "BUILD\n";
    buildJob $job;
    print "DONE\n";
    };
    if ($@) {
    warn $@;
    $db->txn_do(sub {
    $job->busy(0);
    $job->locker($$);
    $job->update;
    });
    }
  • replacement in src/hydra.sql at line 132
    [3.5757][3.5757:5793]()
    priority integer not null,
    [3.5757]
    [2.4969]
    priority integer not null default 0,
  • replacement in src/hydra.sql at line 134
    [2.4970][2.4970:5140]()
    busy integer not null, -- true means someone is building this job now
    locker text not null, -- !!! hostname/pid of the process building this job?
    [2.4970]
    [3.4340]
    busy integer not null default 0, -- true means someone is building this job now
    locker text not null default '', -- !!! hostname/pid of the process building this job?
  • file addition: runner.pl (----------)
    [21.4]
    #! @perl@ -w
    use strict;
    use HydraFrontend::Schema;
    my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
    # Unlock jobs whose building process has died.
    $db->txn_do(sub {
    my @jobs = $db->resultset('Jobs')->search({ busy => 1 });
    foreach my $job (@jobs) {
    my $pid = $job->locker;
    if (kill(0, $pid) != 1) { # see if we can signal the process
    print "job ", $job->id, " pid $pid died, unlocking\n";
    $job->busy(0);
    $job->locker("");
    $job->update;
    }
    }
    });
    while (1) {
    print "looking for runnable jobs...\n";
    my $job;
    $db->txn_do(sub {
    my @jobs = $db->resultset('Jobs')->search({ busy => 0 }, {order_by => ["priority", "timestamp"]});
    print "# of available jobs: ", scalar(@jobs), "\n";
    if (scalar @jobs > 0) {
    $job = $jobs[0];
    $job->busy(1);
    $job->locker($$);
    $job->update;
    }
    });
    # Start the job. We need to do this outside the transaction in
    # case it aborts or something.
    if (defined $job) {
    print "starting job ", $job->id, "\n";
    eval {
    system("perl -I HydraFrontend/lib -w ./build.pl " . $job->id);
    };
    if ($@) {
    warn $@;
    $db->txn_do(sub {
    $job->busy(0);
    $job->locker($$);
    $job->update;
    });
    }
    }
    print "sleeping...\n";
    sleep(10);
    }
  • edit in src/scheduler.pl at line 5
    [3.1372][3.1372:1392]()
    use File::Basename;
  • edit in src/scheduler.pl at line 14
    [3.113][3.113:117](),[3.117][3.5762:5777](),[3.1426][3.5762:5777](),[3.5777][2.5396:5495](),[3.3059][3.1492:1493](),[3.4751][3.1492:1493](),[2.5495][3.1492:1493](),[3.5840][3.1492:1493](),[3.1492][3.1492:1493](),[3.1493][3.5841:6048](),[3.6048][3.2311:2317](),[3.2311][3.2311:2317](),[3.2317][3.2305:2333](),[3.2333][3.0:96](),[3.96][3.2358:2408](),[3.2358][3.2358:2408](),[3.2408][3.118:152](),[3.152][3.2483:2511](),[3.2483][3.2483:2511](),[3.2511][3.2317:2318](),[3.2317][3.2317:2318](),[3.2318][3.2512:2542](),[3.2542][3.6049:6142](),[3.6142][3.2607:2636](),[3.2607][3.2607:2636](),[3.2636][3.2379:2380](),[3.2379][3.2379:2380](),[3.2380][3.97:425](),[3.425][3.2679:2685](),[3.2679][3.2679:2685](),[3.2685][3.2421:2422](),[3.2421][3.2421:2422](),[3.2422][3.6143:6366](),[3.6366][2.5496:5538](),[2.5538][3.6366:6596](),[3.6366][3.6366:6596](),[3.6596][3.4752:4784](),[3.4784][3.6596:6612](),[3.6596][3.6596:6612](),[3.6612][3.0:53](),[3.53][3.6620:6621](),[3.6620][3.6620:6621](),[3.6621][3.2002:2107](),[3.2107][3.5794:5878](),[3.5878][3.4876:5138](),[3.4876][3.4876:5138](),[3.5138][3.5879:5924](),[3.5924][3.5180:5275](),[3.5180][3.5180:5275](),[3.2904][3.3535:3546](),[3.5275][3.3535:3546](),[3.3535][3.3535:3546](),[3.3546][3.54:240](),[3.6621][3.54:240](),[3.240][3.5925:5963](),[3.5963][3.280:414](),[3.280][3.280:414](),[3.414][3.6635:6636](),[3.6635][3.6635:6636](),[3.6636][3.426:456](),[3.448][3.6679:6680](),[3.456][3.6679:6680](),[3.4701][3.6679:6680](),[3.6679][3.6679:6680](),[3.680][3.680:886](),[3.886][3.5964:6010](),[3.6010][3.934:1074](),[3.934][3.934:1074](),[3.1074][3.4702:5208](),[3.5208][3.6011:6057](),[3.6057][3.5308:5434](),[3.5308][3.5308:5434](),[3.5434][3.1074:1120](),[3.1074][3.1074:1120](),[3.1120][3.5435:5541](),[3.5541][3.6058:6100](),[3.6100][3.5633:5774](),[3.5633][3.5633:5774](),[3.1120][3.3586:3610](),[3.5774][3.3586:3610](),[3.3586][3.3586:3610](),[3.3610][3.1121:1138](),[3.1138][3.3616:3617](),[3.3616][3.3616:3617]()
    }
    sub buildJob {
    my ($project, $jobset, $jobName, $description, $drvPath, $outPath, $usedInputs, $system) = @_;
    if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {
    print " already done\n";
    return;
    }
    my $isCachedBuild = 1;
    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
    my $startTime = 0;
    my $stopTime = 0;
    if (!isValidPath($outPath)) {
    $isCachedBuild = 0;
    $startTime = time();
    print " BUILDING\n";
    my $res = system("nix-store --realise $drvPath");
    $stopTime = time();
    $outputCreated = $res == 0;
    }
    my $buildStatus;
    if ($outputCreated) {
    # "Positive" failures, e.g. the builder returned exit code 0
    # but flagged some error condition.
    $buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
    } else {
    $buildStatus = 1; # = Nix failure
    }
    $db->txn_do(sub {
    my $build = $db->resultset('Builds')->create(
    { timestamp => time()
    , project => $project->name
    , jobset => $jobset->name
    , attrname => $jobName
    , description => $description
    , drvpath => $drvPath
    , outpath => $outPath
    , iscachedbuild => $isCachedBuild
    , buildstatus => $buildStatus
    , starttime => $startTime
    , stoptime => $stopTime
    , system => $system
    });
    print " build ID = ", $build->id, "\n";
    foreach my $inputName (keys %{$usedInputs}) {
    my $input = $usedInputs->{$inputName};
    $db->resultset('Inputs')->create(
    { build => $build->id
    , name => $inputName
    , type => $input->{type}
    , uri => $input->{uri}
    #, revision => $input->{orig}->revision
    #, tag => $input->{orig}->tag
    , value => $input->{value}
    , dependency => $input->{id}
    , path => ($input->{storePath} or "") # !!! temporary hack
    });
    }
    my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
    if (-e $logPath) {
    print " LOG $logPath\n";
    $db->resultset('Buildlogs')->create(
    { build => $build->id
    , logphase => "full"
    , path => $logPath
    , type => "raw"
    });
    }
    if ($outputCreated) {
    if (-e "$outPath/log") {
    foreach my $logPath (glob "$outPath/log/*") {
    print " LOG $logPath\n";
    $db->resultset('Buildlogs')->create(
    { build => $build->id
    , logphase => basename($logPath)
    , path => $logPath
    , type => "raw"
    });
    }
    }
    if (-e "$outPath/nix-support/hydra-build-products") {
    open LIST, "$outPath/nix-support/hydra-build-products" or die;
    while (<LIST>) {
    /^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
    my $type = $1;
    my $subtype = $2;
    my $path = $3;
    die unless -e $path;
    $db->resultset('Buildproducts')->create(
    { build => $build->id
    , type => $type
    , subtype => $subtype
    , path => $path
    });
    }
    close LIST;
    } else {
    $db->resultset('Buildproducts')->create(
    { build => $build->id
    , type => "nix-build"
    , subtype => ""
    , path => $outPath
    });
    }
    }
    });
  • replacement in src/scheduler.pl at line 63
    [3.1085][2.5689:5794]()
    buildJob($project, $jobset, $jobName, $description, $drvPath, $outPath, $inputInfo, $job->{system});
    [3.1085]
    [3.1176]
    $db->txn_do(sub {
    if (scalar($db->resultset('Builds')->search(
    { project => $project->name, jobset => $jobset->name
    , attrname => $jobName, outPath => $outPath })) > 0)
    {
    print " already done\n";
    return;
    }
    if (scalar($db->resultset('Jobs')->search(
    { project => $project->name, jobset => $jobset->name
    , attrname => $jobName, outPath => $outPath })) > 0)
    {
    print " already queued\n";
    return;
    }
    print " adding to queue\n";
    my $job = $db->resultset('Jobs')->create(
    { timestamp => time()
    , priority => 0
    , busy => 0
    , locker => ""
    , project => $project->name
    , jobset => $jobset->name
    , attrname => $jobName
    , description => $description
    , drvpath => $drvPath
    , outpath => $outPath
    , system => $job->{system}
    });
    foreach my $inputName (keys %{$inputInfo}) {
    my $input = $inputInfo->{$inputName};
    $db->resultset('Inputs')->create(
    { job => $job->id
    , name => $inputName
    , type => $input->{type}
    , uri => $input->{uri}
    #, revision => $input->{orig}->revision
    #, tag => $input->{orig}->tag
    , value => $input->{value}
    , dependency => $input->{id}
    , path => ($input->{storePath} or "") # !!! temporary hack
    });
    }
    });