* hydra_scheduler: use eval-jobs.

[?]

Mar 9, 2009, 1:04 PM

POPU2ATH2HHBTGHKRAV3EY2K55P664IARI3YJGLDKVJ6PQPXBQ4AC

Dependencies

[2] N6E7R424
[3] XNCWZ7OT * Get the job priority from the meta.schedulingPriority attribute.
[4] M3WSK4CB
[5] 2T42QGZD * Register builds as GC roots so they don't get deleted.
[6] CN5LSNJN * Pass the version of dependencies to the Nix expression.
[7] BHZXGT2H * Channels: provide an index page that lists all the packages in the
[8] KD5237CU * eval-jobs now efficiently evaluates all Hydra jobs from a
[9] YAPITGB3 * Boolean inputs.
[10] PHX2HIVG * Store info about the build inputs in the build record.
[11] CMU3YKOU * Store the release name.
[12] M552HLIA * Support variant builds.
[13] 67P45PY4
[14] HJLYC753 * Adding input value alternatives.
[15] BVOPAMLS
[16] 6BLUKEQ2 * Caching of "path" inputs, and fake a revision number for those.
[17] 3ZCEPLNO
[18] VJHIHMEH * Store the meta.longDescription and meta.license attributes in the
[19] H7CNGK4O * Log evaluation errors etc. in the DB.
[20] HOVPJBFF * Require every argument to be specified.
[21] FMJMW4PU
[22] YGRLM2SK * Export all relevant info about the derivation.
[23] FDE3BJAP * Refactoring.
[24] TLZ2SPBR
[25] ZVTSOVHN * Support Subversion checkouts.
[26] 5QJP6JHS * Get dependencies from the database.
[27] ECBA3GQO * Make the schema class names match the case of the SQL table names.
[28] L2E6EVE2 * Merged the Build and Job tables.
[29] 7YBYT2LQ
[30] TWURROKC
[31] N22GPKYT * Put info about logs / build products in the DB.
[32] KOTB7BKV
[33] X27GNHDV * Basic job info in the database.
[34] 4LWGZL33
[*] 4N5APGRG * Start of a helper tool to evaluate job expressions efficiently.

Change contents

edit in src/c/eval-jobs.cc at line 48

[8.643]→[8.643:644](∅→∅)
edit in src/c/eval-jobs.cc at line 58

[8.1087]→[8.1087:1096](∅→∅)
edit in src/c/eval-jobs.cc at line 59

[8.1102]

[8.1297]
replacement in src/c/eval-jobs.cc at line 102

[9.748]→[9.748:789](∅→∅)

xmlAttrs["name"] = attrPath;

[9.748]

[9.113]

xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
edit in src/c/eval-jobs.cc at line 111

[9.586]

[9.944]

xmlAttrs["schedulingPriority"] = drv.queryMetaInfo(state, "schedulingPriority");
edit in src/c/eval-jobs.cc at line 179

[36.855]

[36.855]

if (releaseExpr == "") throw UsageError("no expression specified");
edit in src/script/hydra_scheduler.pl at line 4

[9.1346]

[9.1346]

use feature 'switch';
replacement in src/script/hydra_scheduler.pl at line 36

[9.2186]→[9.6685:6702](∅→∅),[9.6685]→[9.6685:6702](∅→∅),[9.6702]→[9.5276:5316](∅→∅)

sub fetchInput {
my ($input, $alt, $inputInfo) = @_;

[9.2186]

[9.6736]

sub fetchInputAlt {
my ($input, $alt) = @_;
replacement in src/script/hydra_scheduler.pl at line 93

[9.3744]→[9.2196:2232](∅→∅),[9.2196]→[9.2196:2232](∅→∅)

$$inputInfo{$input->name} =

[9.3744]

[9.2232]

return
replacement in src/script/hydra_scheduler.pl at line 144

[9.5062]→[9.5062:5107](∅→∅)

$$inputInfo{$input->name} =

[9.5062]

[9.5107]

return
replacement in src/script/hydra_scheduler.pl at line 156

[9.5511]→[9.5511:5586](∅→∅)

$$inputInfo{$input->name} = {type => $type, value => $alt->value};

[9.5511]

[9.5586]

return {type => $type, value => $alt->value};
replacement in src/script/hydra_scheduler.pl at line 161

[9.2350]→[9.2350:2425](∅→∅)

$$inputInfo{$input->name} = {type => $type, value => $alt->value};

[9.2350]

[9.2425]

return {type => $type, value => $alt->value};
replacement in src/script/hydra_scheduler.pl at line 170

[9.7161]→[9.51:157](∅→∅),[9.157]→[9.5669:5674](∅→∅),[9.5669]→[9.5669:5674](∅→∅),[9.5674]→[9.158:198](∅→∅),[9.198]→[9.2247:2473](∅→∅),[9.2473]→[9.362:382](∅→∅),[9.362]→[9.362:382](∅→∅)

sub checkJob {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_;

# Instantiate the store derivation.
(my $res, my $drvPath, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res;
chomp $drvPath;

[9.7161]

[9.382]

sub fetchInputs {
my ($jobset, $inputInfo) = @_;
foreach my $input ($jobset->jobsetinputs->all) {
foreach my $alt ($input->jobsetinputalts->all) {
push @{$$inputInfo{$input->name}}, fetchInputAlt($input, $alt);
}
}
}
edit in src/script/hydra_scheduler.pl at line 179

[9.383]→[9.2474:2878](∅→∅)

# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
($res, my $infoXml, $stderr) = captureStdoutStderr(
qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *),
"-f", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot get information about the job `$jobName':\n$stderr" unless $res;
replacement in src/script/hydra_scheduler.pl at line 180

[9.699]→[9.5539:5620](∅→∅),[9.5620]→[9.763:805](∅→∅),[9.763]→[9.763:805](∅→∅)

my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";

[9.699]

[9.805]

sub checkJob {
my ($project, $jobset, $inputInfo, $job) = @_;
replacement in src/script/hydra_scheduler.pl at line 183

[9.806]→[9.5621:5688](∅→∅),[9.5688]→[9.893:1005](∅→∅),[9.893]→[9.893:1005](∅→∅),[9.1005]→[9.3190:3396](∅→∅),[9.3396]→[7.4646:4740](∅→∅),[7.4740]→[9.3396:3401](∅→∅),[9.3396]→[9.3396:3401](∅→∅),[9.3401]→[9.1005:1049](∅→∅),[9.1005]→[9.1005:1049](∅→∅)

my $job = $info->{item}->{$jobName};
die if !defined $job;

my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
my $longDescription = defined $job->{meta}->{longDescription} ? $job->{meta}->{longDescription}->{value} : "";
my $license = defined $job->{meta}->{license} ? $job->{meta}->{license}->{value} : "";
my $homepage = defined $job->{meta}->{homepage} ? $job->{meta}->{homepage}->{value} : "";

die unless $job->{drvPath} eq $drvPath;

[9.806]

[9.1049]

my $jobName = $job->{jobName};
my $drvPath = $job->{drvPath};
replacement in src/script/hydra_scheduler.pl at line 188

[3.24]→[3.24:224](∅→∅)

if (defined $job->{meta}->{schedulingPriority} &&
$job->{meta}->{schedulingPriority}->{value} =~ /^\d+$/)
{
$priority = int($job->{meta}->{schedulingPriority}->{value});
}

[3.24]

[3.224]

$priority = int($job->{schedulingPriority})
if $job->{schedulingPriority} =~ /^\d+$/;
replacement in src/script/hydra_scheduler.pl at line 208

[9.10276]→[9.10276:10318](∅→∅),[9.10318]→[9.3402:3486](∅→∅),[9.3486]→[9.2969:3007](∅→∅),[9.10318]→[9.2969:3007](∅→∅)

, description => $description
, longdescription => $longDescription
, license => $license
, nixname => $job->{name}

[9.10276]

[9.10318]

, description => $job->{description}
, longdescription => $job->{longDescription}
, license => $job->{license}
, nixname => $job->{nixName}
replacement in src/script/hydra_scheduler.pl at line 224

[9.12524]→[9.10442:10545](∅→∅),[9.10442]→[9.10442:10545](∅→∅)

foreach my $inputName (keys %{$inputInfo}) {
my $input = $inputInfo->{$inputName};

[9.12524]

[9.5621]

foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
replacement in src/script/hydra_scheduler.pl at line 228

[9.12614]→[9.10625:10662](∅→∅),[9.10625]→[9.10625:10662](∅→∅)

, name => $inputName

[9.12614]

[9.10662]

, name => $arg->{name}
replacement in src/script/hydra_scheduler.pl at line 238

[9.11037]→[5.528:529](∅→∅)

[9.11037]

[5.529]
replacement in src/script/hydra_scheduler.pl at line 257

[9.1181]→[9.1181:1316](∅→∅)

sub checkJobAlternatives {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_;

[9.1181]

[9.1316]

sub inputsToArgs {
my ($inputInfo) = @_;
my @res = ();
replacement in src/script/hydra_scheduler.pl at line 261

[9.1317]→[9.1317:1356](∅→∅),[9.1356]→[9.6101:6226](∅→∅),[9.6226]→[9.3187:3319](∅→∅),[9.3319]→[9.1451:1512](∅→∅),[9.6249]→[9.1451:1512](∅→∅),[9.1451]→[9.1451:1512](∅→∅),[9.1512]→[9.3320:3379](∅→∅),[9.3379]→[9.1576:1647](∅→∅),[9.1576]→[9.1576:1647](∅→∅),[9.1647]→[9.457:533](∅→∅),[9.533]→[9.1647:1734](∅→∅),[9.1647]→[9.1647:1734](∅→∅),[9.1734]→[9.3380:3477](∅→∅),[9.3477]→[9.3902:3952](∅→∅),[9.3952]→[9.3478:3519](∅→∅),[9.1902]→[9.3478:3519](∅→∅),[9.3519]→[9.1932:1995](∅→∅),[9.1932]→[9.1932:1995](∅→∅),[9.1995]→[9.3953:4003](∅→∅),[9.4003]→[9.1955:2051](∅→∅),[9.2051]→[6.0:99](∅→∅),[6.99]→[9.4080:4159](∅→∅),[9.2051]→[9.4080:4159](∅→∅),[9.4080]→[9.4080:4159](∅→∅),[9.4159]→[9.2437:2504](∅→∅),[9.2504]→[9.3628:3714](∅→∅),[9.2192]→[9.3628:3714](∅→∅),[9.3714]→[9.2505:2656](∅→∅)

if ($n >= scalar @{$argsNeeded}) {
eval {
checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs);
};
if ($@) {
print "error evaluating job `", $jobName, "': $@";
setJobsetError($jobset, $@);
}
return;
}

my $argName = @{$argsNeeded}[$n];
#print "finding alternatives for argument $argName\n";

my ($input) = $jobset->jobsetinputs->search({name => $argName});

my %newInputInfo = %{$inputInfo}; $inputInfo = \%newInputInfo; # clone

if (defined $input) {

foreach my $alt ($input->jobsetinputalts) {
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo);
my @newArgs = @{$extraArgs};
if (defined $inputInfo->{$argName}->{storePath}) {
push @newArgs, "--arg", $argName,
"{path = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" outPath = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
} elsif ($inputInfo->{$argName}->{type} eq "string") {
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
} elsif ($inputInfo->{$argName}->{type} eq "boolean") {
push @newArgs, "--arg", $argName, $inputInfo->{$argName}->{value};

[9.1317]

[9.2283]

foreach my $input (keys %{$inputInfo}) {
foreach my $alt (@{$inputInfo->{$input}}) {
given ($alt->{type}) {
when ("string") {
push @res, "--argstr", $input, $alt->{value};
}
when ("boolean") {
push @res, "--arg", $input, $alt->{value};
}
when (["svn", "path"]) {
push @res, "--arg", $input, (
"{ outPath = builtins.storePath " . $alt->{storePath} . "" .
"; rev = \"" . $alt->{revision} . "\"" .
";}"
);
}
edit in src/script/hydra_scheduler.pl at line 278

[9.2297]→[9.2297:2392](∅→∅),[9.2392]→[9.3715:3784](∅→∅)

checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
edit in src/script/hydra_scheduler.pl at line 280

[9.2489]→[9.2489:2562](∅→∅),[9.2562]→[9.12615:12814](∅→∅)

else {

(my $prevBuild) = $db->resultset('Builds')->search(
{finished => 1, project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
{join => 'resultInfo', order_by => "timestamp DESC", rows => 1});
replacement in src/script/hydra_scheduler.pl at line 281

[9.2725]→[9.2725:3365](∅→∅),[9.3365]→[6.100:209](∅→∅),[6.209]→[9.3365:3366](∅→∅),[9.3365]→[9.3365:3366](∅→∅),[9.3366]→[6.210:366](∅→∅),[6.366]→[9.3785:3822](∅→∅),[9.3366]→[9.3785:3822](∅→∅),[9.3822]→[6.367:651](∅→∅),[6.651]→[9.3905:3914](∅→∅),[9.2154]→[9.3905:3914](∅→∅),[9.3905]→[9.3905:3914](∅→∅),[9.3914]→[9.3366:3453](∅→∅),[9.3366]→[9.3366:3453](∅→∅),[9.3453]→[9.3915:3980](∅→∅),[9.3980]→[9.3603:3609](∅→∅),[9.3603]→[9.3603:3609](∅→∅)

if (!defined $prevBuild) {
# !!! reschedule?
die "missing input `$argName'";
}

# The argument name matches a previously built job in this
# jobset. Pick the most recent build. !!! refine the
# selection criteria: e.g., most recent successful build.
if (!isValidPath($prevBuild->outpath)) {
die "input path " . $prevBuild->outpath . " has been garbage-collected";
}

$$inputInfo{$argName} =
{ type => "build"
, storePath => $prevBuild->outpath
, id => $prevBuild->id
};

my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";
my $versionRE = "(?:[A-Za-z0-9\.\-]+)";

my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);
my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;

my @newArgs = @{$extraArgs};
push @newArgs, "--arg", $argName,
"{ path = builtins.storePath " . $prevBuild->outpath . "; " .
" outPath = builtins.storePath " . $prevBuild->outpath . "; " .
($version ? " version = \"$version\"; " : "") . # !!! escape
"}";

checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}

[9.2725]

[9.3609]

return @res;
edit in src/script/hydra_scheduler.pl at line 293

[9.2121]→[9.3692:3832](∅→∅),[9.3692]→[9.3692:3832](∅→∅),[9.3832]→[4.118:195](∅→∅),[4.195]→[9.3867:3868](∅→∅),[9.3867]→[9.3867:3868](∅→∅),[9.3868]→[4.196:328](∅→∅)

# Fetch the input containing the Nix expression.
(my $exprInput) = $jobset->jobsetinputs->search({name => $jobset->nixexprinput});
die "No input named " . $jobset->nixexprinput unless defined $exprInput;

die "Multiple alternatives for the Nix expression input not supported yet"
if scalar($exprInput->jobsetinputalts) != 1;
replacement in src/script/hydra_scheduler.pl at line 294

[9.3942]→[9.3942:4018](∅→∅)

fetchInput($exprInput, $exprInput->jobsetinputalts->first, $inputInfo);

[9.3942]

[9.4018]

# Fetch all values for all inputs.
fetchInputs($jobset, $inputInfo);
replacement in src/script/hydra_scheduler.pl at line 297

[9.4019]→[9.4019:4054](∅→∅),[9.4054]→[9.7481:7582](∅→∅),[9.7481]→[9.7481:7582](∅→∅)

# Evaluate the Nix expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;

[9.4019]

[9.3983]

# Evaluate the job expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->[0]->{storePath}
or die "cannot find the input containing the job expression";
$nixExprPath .= "/" . $jobset->nixexprpath;
edit in src/script/hydra_scheduler.pl at line 302

[9.3984]→[9.3984:4023](∅→∅),[9.4023]→[9.7582:7583](∅→∅),[9.7582]→[9.7582:7583](∅→∅)

print "evaluating $nixExprPath\n";
replacement in src/script/hydra_scheduler.pl at line 303

[9.4086]→[9.4086:4164](∅→∅)

"nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml");

[9.4086]

[9.4164]

"eval-jobs", $nixExprPath, inputsToArgs($inputInfo));
replacement in src/script/hydra_scheduler.pl at line 307

[9.7841]→[2.0:54](∅→∅),[2.54]→[9.7889:8027](∅→∅),[9.7889]→[9.7889:8027](∅→∅)

ForceArray => ['value', 'attr'],
KeyAttr => ['name'],
SuppressEmpty => '',
ValueAttr => [value => 'value'])

[9.7841]

[9.8027]

ForceArray => ['error', 'job', 'arg'],
KeyAttr => [],
SuppressEmpty => '')
replacement in src/script/hydra_scheduler.pl at line 312

[9.8070]→[9.8070:8109](∅→∅)

die unless defined $jobs->{attrs};

[9.8070]

[9.8109]

# Store the errors messages for jobs that failed to evaluate.
foreach my $error (@{$jobs->{error}}) {
print "error at " . $error->{location} . ": " . $error->{msg} . "\n";
}
replacement in src/script/hydra_scheduler.pl at line 317

[9.8110]→[9.4055:4312](∅→∅),[9.4312]→[9.8110:8170](∅→∅),[9.8110]→[9.8110:8170](∅→∅),[9.8170]→[9.4253:4297](∅→∅),[9.4297]→[9.8206:8207](∅→∅),[9.8206]→[9.8206:8207](∅→∅),[9.8207]→[9.4313:4351](∅→∅),[9.4351]→[9.8207:8265](∅→∅),[9.8207]→[9.8207:8265](∅→∅),[9.3657]→[9.8294:8295](∅→∅),[9.8294]→[9.8294:8295](∅→∅),[9.8295]→[9.4352:4437](∅→∅),[9.4437]→[9.8413:8469](∅→∅),[9.8413]→[9.8413:8469](∅→∅),[9.3671]→[9.8469:8555](∅→∅),[9.8469]→[9.8469:8555](∅→∅),[9.8555]→[9.4298:4347](∅→∅),[9.4347]→[9.4438:4508](∅→∅),[9.8609]→[9.4438:4508](∅→∅),[9.4508]→[9.8811:8821](∅→∅),[9.8811]→[9.8811:8821](∅→∅),[9.9062]→[9.9062:9063](∅→∅),[9.9063]→[9.6250:6352](∅→∅),[9.6352]→[9.4348:4406](∅→∅),[9.4406]→[9.6410:6421](∅→∅),[9.6410]→[9.6410:6421](∅→∅),[9.6421]→[9.2657:2785](∅→∅)

# Iterate over the attributes listed in the Nix expression and
# perform the builds described by them. If an attribute is a
# function, then fill in the function arguments with the
# (alternative) values supplied in the jobsetinputs table.
foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
print "considering job $jobName\n";

my @argsNeeded = ();

my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};

# !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
if (defined $jobExpr->{function}->{attrspat}) {
foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
#print "needs input $argName\n";
push @argsNeeded, $argName;
}
}

eval {
checkJobAlternatives(
$project, $jobset, {}, $nixExprPath,
$jobName, $jobExpr, [], \@argsNeeded, 0);
};
if ($@) {
print "error checking job ", $jobName, ": $@";
setJobsetError($jobset, $@);
}

[9.8110]

[9.7080]

# Schedule each successfully evaluated job.
foreach my $job (@{$jobs->{job}}) {
print "considering job " . $job->{jobName} . "\n";
checkJob($project, $jobset, $inputInfo, $job);