* Implemented the clone feature.

[?]
Oct 26, 2009, 5:01 PM
3XTHEUMP2ZOMPQWE3S5QWHIHCEJNEXGDPQB3JUVZFPS3RFMY455QC

Dependencies

  • [2] QNDXPVCI * Store the jobset's nixExprPath and nixExprInput fields in a build to
  • [3] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
  • [4] TIOBBINA * Some renaming.
  • [5] WRIU3S5E * UI for cloning builds (not functional yet).
  • [6] IXCUNELF * Don't bother with the Jobs.active column anymore.
  • [7] KOTB7BKV
  • [8] LBNVQXUB * Build the /build stuff in a separate controller.
  • [9] N22GPKYT * Put info about logs / build products in the DB.
  • [10] TLZ2SPBR
  • [11] BVOPAMLS
  • [12] XNCWZ7OT * Get the job priority from the meta.schedulingPriority attribute.
  • [13] JK2QWPH6
  • [14] NLJJZVHO * Use ->update({...}) properly.
  • [15] LZO3C2KI * Hack around those SQLite timeouts: just retry the transaction.
  • [16] X27GNHDV * Basic job info in the database.
  • [17] L2E6EVE2 * Merged the Build and Job tables.
  • [18] ECBA3GQO * Make the schema class names match the case of the SQL table names.
  • [19] SHZLOM5M * eval-jobs -> hydra_eval_jobs.
  • [20] RWIBJ5L4 * Autoflush stdout.
  • [21] 5MNUNZWR * Store meta.maintainers.
  • [22] 5QJP6JHS * Get dependencies from the database.
  • [23] 6BLUKEQ2 * Caching of "path" inputs, and fake a revision number for those.
  • [24] 2T42QGZD * Register builds as GC roots so they don't get deleted.
  • [25] GNIEG2GC * Disambiguate jobs by jobset name. I.e. jobs with the same name in
  • [26] POPU2ATH * hydra_scheduler: use eval-jobs.
  • [27] FDE3BJAP * Refactoring.
  • [28] 7ZHHVD6Q * Inputs of type "build" must now be declared explicitly.
  • [29] N6E7R424
  • [30] AHTEIK7G * Added a maintainers field to the Builds table.
  • [31] VJHIHMEH * Store the meta.longDescription and meta.license attributes in the
  • [32] S6OISBQ3 * Mark the "current" builds in a jobset, i.e. those corresponding to
  • [33] ZP3M3JWR * Be a bit less aggressive in rescheduling builds that have already
  • [34] BTOXLRG3 * Record the input containing the Nix expression (release.nix) in the
  • [35] AFTXA575 * $HYDRA_DATA environment variable.
  • [36] T4LLYESZ * Nix expression for building Hydra.
  • [37] NI5BVF2V * In job inputs of type "build", allow the project and jobset names of
  • [38] IWHFLFVV * Randomly permute the order in which builds are added. This is
  • [39] M552HLIA * Support variant builds.
  • [40] TULPZ62Y * Perform builds in parallel.
  • [41] YAPITGB3 * Boolean inputs.
  • [42] A63IHCMX * Register GC roots properly.
  • [43] CVDK3XJK * In the scheduler, don't check if we've already done a build (except
  • [44] H7CNGK4O * Log evaluation errors etc. in the DB.
  • [45] S5PV6IIM * Represent jobs explicitly in the DB.
  • [46] 7YBYT2LQ

Change contents

  • replacement in src/lib/Hydra/Controller/Build.pm at line 407
    [3.32][3.32:124]()
    my ($nixExprPath, $nixExprInput) = Hydra::Controller::Jobset::nixExprPathFromParams $c;
    [3.32]
    [5.409]
    my ($nixExprPath, $nixExprInputName) = Hydra::Controller::Jobset::nixExprPathFromParams $c;
  • edit in src/lib/Hydra/Controller/Build.pm at line 412
    [3.258]
    [3.258]
    my $inputInfo = {};
  • replacement in src/lib/Hydra/Controller/Build.pm at line 422
    [3.656][3.656:680]()
    fetchInput(
    [3.656]
    [3.680]
    # !!! fetchInput can take a long time, which might cause
    # the current HTTP request to time out. So maybe this
    # should be done asynchronously. But then error reporting
    # becomes harder.
    my $info = fetchInput(
  • edit in src/lib/Hydra/Controller/Build.pm at line 429
    [3.800]
    [3.800]
    push @{$$inputInfo{$inputName}}, $info if defined $info;
  • replacement in src/lib/Hydra/Controller/Build.pm at line 434
    [3.847][5.410:471](),[5.410][5.410:471]()
    $c->flash->{buildMsg} = "Build XXX added to the queue.";
    [3.847]
    [5.471]
    my ($jobs, $nixExprInput) = evalJobs($inputInfo, $nixExprInputName, $nixExprPath);
    my $job;
    foreach my $j (@{$jobs->{job}}) {
    print STDERR $j->{jobName}, "\n";
    if ($j->{jobName} eq $jobName) {
    error($c, "Nix expression returned multiple builds for job $jobName.")
    if $job;
    $job = $j;
    }
    }
    error($c, "Nix expression did not return a job named $jobName.") unless $job;
    my %currentBuilds;
    my $newBuild = checkBuild(
    $c->model('DB'), $build->project, $build->jobset,
    $inputInfo, $nixExprInput, $job, \%currentBuilds);
    error($c, "This build has already been performed.") unless $newBuild;
    $c->flash->{buildMsg} = "Build " . $newBuild->id . " added to the queue.";
  • edit in src/lib/Hydra/Helper/AddBuilds.pm at line 4
    [3.2711]
    [3.2711]
    use feature 'switch';
    use XML::Simple;
  • edit in src/lib/Hydra/Helper/AddBuilds.pm at line 7
    [3.2735]
    [3.2735]
    use IPC::Run;
  • replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 11
    [3.2785][3.2785:2815]()
    our @EXPORT = qw(fetchInput);
    [3.2785]
    [3.2815]
    our @EXPORT = qw(fetchInput evalJobs checkBuild);
  • edit in src/lib/Hydra/Helper/AddBuilds.pm at line 229
    [3.10397]
    [3.10397]
    }
    sub inputsToArgs {
    my ($inputInfo) = @_;
    my @res = ();
    foreach my $input (keys %{$inputInfo}) {
    foreach my $alt (@{$inputInfo->{$input}}) {
    given ($alt->{type}) {
    when ("string") {
    push @res, "--argstr", $input, $alt->{value};
    }
    when ("boolean") {
    push @res, "--arg", $input, $alt->{value};
    }
    when (["svn", "path", "build"]) {
    push @res, "--arg", $input, (
    "{ outPath = builtins.storePath " . $alt->{storePath} . "" .
    (defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .
    (defined $alt->{version} ? "; version = \"" . $alt->{version} . "\"" : "") .
    ";}"
    );
    }
    }
    }
    }
    return @res;
    }
    sub captureStdoutStderr {
    my $stdin = ""; my $stdout; my $stderr;
    my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr);
    return ($res, $stdout, $stderr);
  • edit in src/lib/Hydra/Helper/AddBuilds.pm at line 266
    [3.10399]
    sub evalJobs {
    my ($inputInfo, $nixExprInputName, $nixExprPath) = @_;
    my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
    or die "Cannot find the input containing the job expression.\n";
    die "Multiple alternatives for the input containing the Nix expression are not supported.\n"
    if scalar @{$inputInfo->{$nixExprInputName}} != 1;
    my $nixExprFullPath = $nixExprInput->{storePath} . "/" . $nixExprPath;
    (my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
    "hydra_eval_jobs", $nixExprFullPath, "--gc-roots-dir", getGCRootsDir,
    inputsToArgs($inputInfo));
    die "Cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
    print STDERR "$stderr";
    my $jobs = XMLin(
    $jobsXml,
    ForceArray => ['error', 'job', 'arg'],
    KeyAttr => [],
    SuppressEmpty => '')
    or die "cannot parse XML output";
    return ($jobs, $nixExprInput);
    }
    # Check whether to add the build described by $buildInfo.
    sub checkBuild {
    my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $currentBuilds) = @_;
    my $jobName = $buildInfo->{jobName};
    my $drvPath = $buildInfo->{drvPath};
    my $outPath = $buildInfo->{outPath};
    my $priority = 100;
    $priority = int($buildInfo->{schedulingPriority})
    if $buildInfo->{schedulingPriority} =~ /^\d+$/;
    my $build;
    txn_do($db, sub {
    # Update the last evaluation time in the database.
    my $job = $jobset->jobs->update_or_create(
    { name => $jobName
    , lastevaltime => time
    });
    $job->update({firstevaltime => time})
    unless defined $job->firstevaltime;
    # Don't add a build that has already been scheduled for this
    # job, or has been built but is still a "current" build for
    # this job. Note that this means that if the sources of a job
    # are changed from A to B and then reverted to A, three builds
    # will be performed (though the last one will probably use the
    # cached result from the first). This ensures that the builds
    # with the highest ID will always be the ones that we want in
    # the channels.
    # !!! Checking $outPath doesn't take meta-attributes into
    # account. For instance, do we want a new build to be
    # scheduled if the meta.maintainers field is changed?
    my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1});
    if (scalar(@previousBuilds) > 0) {
    print STDERR "already scheduled/built\n";
    $currentBuilds->{$_->id} = 1 foreach @previousBuilds;
    return;
    }
    # Nope, so add it.
    $build = $job->builds->create(
    { finished => 0
    , timestamp => time()
    , description => $buildInfo->{description}
    , longdescription => $buildInfo->{longDescription}
    , license => $buildInfo->{license}
    , homepage => $buildInfo->{homepage}
    , maintainers => $buildInfo->{maintainers}
    , nixname => $buildInfo->{nixName}
    , drvpath => $drvPath
    , outpath => $outPath
    , system => $buildInfo->{system}
    , iscurrent => 1
    , nixexprinput => $jobset->nixexprinput
    , nixexprpath => $jobset->nixexprpath
    });
    print STDERR "added to queue as build ", $build->id, "\n";
    $currentBuilds->{$build->id} = 1;
    $build->create_related('buildschedulinginfo',
    { priority => $priority
    , busy => 0
    , locker => ""
    });
    my %inputs;
    $inputs{$jobset->nixexprinput} = $nixExprInput;
    foreach my $arg (@{$buildInfo->{arg}}) {
    $inputs{$arg->{name}} = $inputInfo->{$arg->{name}}->[$arg->{altnr}]
    || die "invalid input";
    }
    foreach my $name (keys %inputs) {
    my $input = $inputs{$name};
    $build->buildinputs_builds->create(
    { name => $name
    , type => $input->{type}
    , uri => $input->{uri}
    , revision => $input->{revision}
    , value => $input->{value}
    , dependency => $input->{id}
    , path => $input->{storePath} || "" # !!! temporary hack
    , sha256hash => $input->{sha256hash}
    });
    }
    });
    return $build;
    };
  • edit in src/script/hydra_scheduler.pl at line 5
    [5.289][5.1346:1363](),[5.1346][5.1346:1363]()
    use XML::Simple;
  • edit in src/script/hydra_scheduler.pl at line 8
    [5.1770][5.1980:1994](),[3.10946][5.1980:1994](),[5.492][5.1980:1994]()
    use IPC::Run;
  • edit in src/script/hydra_scheduler.pl at line 14
    [5.1425][5.1425:1426](),[5.1996][5.1996:2168]()
    sub captureStdoutStderr {
    my $stdin = ""; my $stdout; my $stderr;
    my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr);
    return ($res, $stdout, $stderr);
    }
  • edit in src/script/hydra_scheduler.pl at line 15
    [5.6685][5.2169:2174]()
  • edit in src/script/hydra_scheduler.pl at line 24
    [5.740][5.382:383](),[5.382][5.382:383](),[5.2878][5.698:699](),[5.698][5.698:699](),[5.699][4.0:163](),[5.66][5.805:806](),[4.163][5.805:806](),[5.807][5.805:806](),[5.3560][5.805:806](),[5.805][5.805:806](),[5.806][4.164:287](),[4.287][5.1084:1085](),[5.1084][5.1084:1085](),[5.1085][5.0:24](),[5.24][4.288:398](),[4.398][5.224:225](),[5.977][5.224:225](),[5.224][5.224:225](),[5.225][5.1393:1415](),[5.1415][5.76:135](),[5.135][4.399:450](),[4.450][5.5255:5286](),[5.5255][5.5255:5286](),[5.5312][5.5312:5363]()
    # Check whether to add the build described by $buildInfo.
    sub checkBuild {
    my ($project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $currentBuilds) = @_;
    my $jobName = $buildInfo->{jobName};
    my $drvPath = $buildInfo->{drvPath};
    my $outPath = $buildInfo->{outPath};
    my $priority = 100;
    $priority = int($buildInfo->{schedulingPriority})
    if $buildInfo->{schedulingPriority} =~ /^\d+$/;
    txn_do($db, sub {
    # Update the last evaluation time in the database.
    my $job = $jobset->jobs->update_or_create(
    { name => $jobName
    , lastevaltime => time
    });
  • edit in src/script/hydra_scheduler.pl at line 25
    [5.5364][4.451:545](),[4.545][5.5466:5467](),[5.5466][5.5466:5467](),[5.5467][5.63:578](),[5.578][5.0:66](),[5.66][5.644:769](),[5.644][5.644:769](),[5.769][4.546:636](),[5.161][5.4099:4142](),[4.636][5.4099:4142](),[5.863][5.4099:4142](),[5.4099][5.4099:4142](),[5.4142][5.864:997](),[5.997][5.9666:9676](),[5.5005][5.9666:9676](),[5.9666][5.9666:9676](),[5.9676][5.998:1007](),[5.1007][5.5600:5627](),[5.9677][5.5600:5627](),[5.5627][4.637:679](),[4.679][5.12279:12341](),[5.5674][5.12279:12341](),[5.12279][5.12279:12341](),[5.12341][4.680:996](),[4.996][5.10318:10386](),[5.1166][5.10318:10386](),[5.3007][5.10318:10386](),[5.10318][5.10318:10386](),[5.10386][4.997:1042](),[4.1042][5.5006:5035](),[5.10425][5.5006:5035](),[5.5035][2.490:592](),[2.592][5.10425:10441](),[5.5035][5.10425:10441](),[5.10425][5.10425:10441]()
    $job->update({firstevaltime => time})
    unless defined $job->firstevaltime;
    # Don't add a build that has already been scheduled for this
    # job, or has been built but is still a "current" build for
    # this job. Note that this means that if the sources of a job
    # are changed from A to B and then reverted to A, three builds
    # will be performed (though the last one will probably use the
    # cached result from the first). This ensures that the builds
    # with the highest ID will always be the ones that we want in
    # the channels.
    # !!! Checking $outPath doesn't take meta-attributes into
    # account. For instance, do we want a new build to be
    # scheduled if the meta.maintainers field is changed?
    my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1});
    if (scalar(@previousBuilds) > 0) {
    print "already scheduled/built\n";
    $currentBuilds->{$_->id} = 1 foreach @previousBuilds;
    return;
    }
    # Nope, so add it.
    my $build = $job->builds->create(
    { finished => 0
    , timestamp => time()
    , description => $buildInfo->{description}
    , longdescription => $buildInfo->{longDescription}
    , license => $buildInfo->{license}
    , homepage => $buildInfo->{homepage}
    , maintainers => $buildInfo->{maintainers}
    , nixname => $buildInfo->{nixName}
    , drvpath => $drvPath
    , outpath => $outPath
    , system => $buildInfo->{system}
    , iscurrent => 1
    , nixexprinput => $jobset->nixexprinput
    , nixexprpath => $jobset->nixexprpath
    });
  • edit in src/script/hydra_scheduler.pl at line 26
    [5.10442][5.5036:5156](),[5.5156][5.17990:18044](),[5.10442][5.17990:18044](),[5.18044][5.5722:5758](),[5.5722][5.5722:5758](),[5.262][5.12456:12524](),[5.5758][5.12456:12524](),[5.12456][5.12456:12524](),[5.12524][5.67:143](),[5.143][4.1043:1092](),[4.1092][5.144:357](),[5.1210][5.144:357](),[5.357][5.5759:5807](),[5.1302][5.5759:5807](),[5.5807][5.358:390](),[5.390][5.10662:10742](),[5.1342][5.10662:10742](),[5.5846][5.10662:10742](),[5.10662][5.10662:10742](),[5.10742][5.3852:3901](),[5.3901][5.10844:10932](),[5.10844][5.10844:10932](),[5.10932][5.391:464](),[5.464][5.2396:2449](),[5.11007][5.2396:2449](),[5.2449][5.11007:11037](),[5.11007][5.11007:11037](),[5.642][5.11037:11045](),[5.11037][5.11037:11045](),[5.5794][5.1176:1179](),[5.11045][5.1176:1179](),[5.1176][5.1176:1179](),[5.1179][5.2962:2964]()
    print "added to queue as build ", $build->id, "\n";
    $currentBuilds->{$build->id} = 1;
    $build->create_related('buildschedulinginfo',
    { priority => $priority
    , busy => 0
    , locker => ""
    });
    my %inputs;
    $inputs{$jobset->nixexprinput} = $nixExprInput;
    foreach my $arg (@{$buildInfo->{arg}}) {
    $inputs{$arg->{name}} = $inputInfo->{$arg->{name}}->[$arg->{altnr}]
    || die "invalid input";
    }
    foreach my $name (keys %inputs) {
    my $input = $inputs{$name};
    $build->buildinputs_builds->create(
    { name => $name
    , type => $input->{type}
    , uri => $input->{uri}
    , revision => $input->{revision}
    , value => $input->{value}
    , dependency => $input->{id}
    , path => $input->{storePath} || "" # !!! temporary hack
    , sha256hash => $input->{sha256hash}
    });
    }
    });
    };
  • edit in src/script/hydra_scheduler.pl at line 36
    [5.1181][5.1353:1416](),[5.1416][5.1316:1317](),[5.1316][5.1316:1317](),[5.1317][5.1417:1783](),[5.1783][5.1412:1462](),[5.1462][5.1824:1959](),[5.1824][5.1824:1959](),[5.1959][5.1463:1663](),[5.1663][5.2024:2094](),[5.2024][5.2024:2094](),[5.2094][5.2283:2297](),[5.2656][5.2283:2297](),[5.3714][5.2283:2297](),[5.2283][5.2283:2297](),[5.3784][5.2473:2489](),[5.2473][5.2473:2489](),[5.12814][5.2724:2725](),[5.2724][5.2724:2725](),[5.2725][5.2095:2112](),[5.2112][5.3609:3611](),[5.3609][5.3609:3611](),[5.3611][5.7480:7481](),[5.7480][5.7480:7481](),[5.7481][5.3981:3982]()
    sub inputsToArgs {
    my ($inputInfo) = @_;
    my @res = ();
    foreach my $input (keys %{$inputInfo}) {
    foreach my $alt (@{$inputInfo->{$input}}) {
    given ($alt->{type}) {
    when ("string") {
    push @res, "--argstr", $input, $alt->{value};
    }
    when ("boolean") {
    push @res, "--arg", $input, $alt->{value};
    }
    when (["svn", "path", "build"]) {
    push @res, "--arg", $input, (
    "{ outPath = builtins.storePath " . $alt->{storePath} . "" .
    (defined $alt->{revision} ? "; rev = \"" . $alt->{revision} . "\"" : "") .
    (defined $alt->{version} ? "; version = \"" . $alt->{version} . "\"" : "") .
    ";}"
    );
    }
    }
    }
    }
    return @res;
    }
  • replacement in src/script/hydra_scheduler.pl at line 54
    [5.544][5.544:608](),[5.608][5.2302:2372](),[5.2302][5.2302:2372](),[5.2372][5.1443:1600](),[5.1600][5.609:688](),[5.2372][5.609:688](),[5.688][5.3983:3984](),[5.2420][5.3983:3984](),[5.7582][5.3983:3984](),[5.7583][5.4024:4086](),[5.4086][5.1164:1273](),[5.687][5.4164:4252](),[5.1273][5.4164:4252](),[5.2483][5.4164:4252](),[5.4164][5.4164:4252](),[5.4252][5.1274:1303]()
    my $nixExprInput = $inputInfo->{$jobset->nixexprinput}->[0]
    or die "cannot find the input containing the job expression";
    die "multiple alternatives for the input containing the Nix expression are not supported"
    if scalar @{$inputInfo->{$jobset->nixexprinput}} != 1;
    my $nixExprPath = $nixExprInput->{storePath} . "/" . $jobset->nixexprpath;
    (my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
    "hydra_eval_jobs", $nixExprPath, "--gc-roots-dir", getGCRootsDir,
    inputsToArgs($inputInfo));
    die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
    print STDERR "$stderr";
    [5.2226]
    [5.7781]
    my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
  • edit in src/script/hydra_scheduler.pl at line 56
    [5.7810][5.7810:7841](),[5.7841][5.2484:2622](),[5.2622][5.8027:8069](),[5.8027][5.8027:8069](),[5.2817][5.8109:8110](),[5.8109][5.8109:8110]()
    my $jobs = XMLin($jobsXml,
    ForceArray => ['error', 'job', 'arg'],
    KeyAttr => [],
    SuppressEmpty => '')
    or die "cannot parse XML output";
  • replacement in src/script/hydra_scheduler.pl at line 61
    [5.2965][4.1093:1182]()
    checkBuild($project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);
    [5.2965]
    [5.7080]
    checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds);