* Refactoring.

[?]

Nov 7, 2008, 2:51 PM

FDE3BJAPDEP3BYT5A5GEGLNXPPZLA2KTGXB4ZNYRP4LJ7IFRKYXAC

Dependencies

[2] M552HLIA * Support variant builds.
[3] GWCV3TQV * BuildInputs table: link to dependencies, include store paths.
[4] PHX2HIVG * Store info about the build inputs in the build record.
[5] IMY5UQE3
[6] N22GPKYT * Put info about logs / build products in the DB.
[7] X27GNHDV * Basic job info in the database.
[8] 5QJP6JHS * Get dependencies from the database.

Change contents

replacement in src/scheduler.pl at line 123

[3.6956]→[3.6956:7002](∅→∅)

print " copied to $storePath\n";

[3.6956]

[2.5347]

print " copied to $storePath\n";
replacement in src/scheduler.pl at line 138

[3.7161]→[2.5598:5669](∅→∅)

sub checkJobSetInstance {
my ($project, $jobset, $inputInfo) = @_;

[3.7161]

[2.5669]

sub checkJob {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_;
replacement in src/scheduler.pl at line 141

[2.5674]→[3.7420:7480](∅→∅),[3.7420]→[3.7420:7480](∅→∅)

die unless defined $inputInfo->{$jobset->nixexprinput};

[2.5674]

[3.7480]

# Instantiate the store derivation.
my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs`
or die "cannot evaluate the Nix expression containing the job definitions: $?";
chomp $drvPath;

# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs`
or die "cannot get information about the job: $?";

my $info = XMLin($infoXml, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";

my $job = $info->{item};
die if !defined $job || $job->{attrPath} ne $jobName;

my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
die unless $job->{drvPath} eq $drvPath;
my $outPath = $job->{outPath};

buildJob($project, $jobset, $jobName, $drvPath, $outPath, $inputInfo, $job->{system});
};

sub checkJobAlternatives {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_;

if ($n >= scalar @{$argsNeeded}) {
checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs);
return;
}

my $argName = @{$argsNeeded}[$n];
print " finding alternatives for argument $argName\n";

my ($input) = $jobset->jobsetinputs->search({name => $argName});

if (defined $input) {

foreach my $alt ($input->jobsetinputalts) {
print " INPUT ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo); # !!! caching
my $newArgs = "";
if (defined $inputInfo->{$argName}->{storePath}) {
# !!! escaping
$newArgs = " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'";
} elsif (defined $inputInfo->{$argName}->{value}) {
$newArgs = " --argstr $argName '" . $inputInfo->{$argName}->{value} . "'";
}
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, $extraArgs . $newArgs, $argsNeeded, $n + 1);
}
}

else {

(my $prevBuild) = $db->resultset('Builds')->search(
{project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
{order_by => "timestamp DESC", rows => 1});

if (!defined $prevBuild) {
# !!! reschedule?
die "missing input `$argName'";
}

# The argument name matches a previously built job in this
# jobset. Pick the most recent build. !!! refine the
# selection criteria: e.g., most recent successful build.
if (!isValidPath($prevBuild->outpath)) {
die "input path " . $prevBuild->outpath . " has been garbage-collected";
}

$$inputInfo{$argName} =
{ type => "build"
, storePath => $prevBuild->outpath
, id => $prevBuild->id
};

checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr,
$extraArgs . " --arg $argName '{path = " . $prevBuild->outpath . ";}'",
$argsNeeded, $n + 1);
}
}
edit in src/scheduler.pl at line 227

[3.7481]

[3.7481]

sub checkJobSet {
my ($project, $jobset) = @_;
my $inputInfo = {};

# Fetch the input containing the Nix expression.
(my $exprInput) = $jobset->jobsetinputs->search({name => $jobset->nixexprinput});
die unless defined $exprInput;

die "not supported yet" if scalar($exprInput->jobsetinputalts) != 1;

fetchInput($exprInput, $exprInput->jobsetinputalts->first, $inputInfo);

# Evaluate the Nix expression.
edit in src/scheduler.pl at line 248

[3.7782]→[3.7782:7810](∅→∅)

#print "$jobsXml";
edit in src/scheduler.pl at line 257

[3.8110]

[3.8110]

# Iterate over the attributes listed in the Nix expression and
# perform the builds described by them. If an attribute is a
# function, then fill in the function arguments with the
# (alternative) values supplied in the jobsetinputs table.
edit in src/scheduler.pl at line 264

[3.8207]

[3.8207]

my @argsNeeded = ();
edit in src/scheduler.pl at line 267

[3.8265]→[3.8265:8294](∅→∅),[3.8294]→[3.3627:3657](∅→∅)

my $extraArgs = "";

my $usedInputs = {};
replacement in src/scheduler.pl at line 268

[3.8295]→[3.8295:8413](∅→∅)

# If the expression is a function, then look at its formal
# arguments and see if we can supply them.

[3.8295]

[3.8413]

# !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
edit in src/scheduler.pl at line 270

[3.8469]→[3.3658:3671](∅→∅)
replacement in src/scheduler.pl at line 272

[3.184]→[3.3690:3707](∅→∅),[3.3707]→[3.184:296](∅→∅),[3.184]→[3.184:296](∅→∅),[3.366]→[3.3708:3777](∅→∅),[3.3777]→[2.5675:6110](∅→∅),[2.6110]→[3.3777:3819](∅→∅),[3.3777]→[3.3777:3819](∅→∅),[3.3819]→[3.391:463](∅→∅),[3.391]→[3.391:463](∅→∅),[3.463]→[3.450:568](∅→∅),[3.568]→[3.563:631](∅→∅),[3.563]→[3.563:631](∅→∅),[3.631]→[2.6111:6147](∅→∅),[2.6147]→[3.3820:3841](∅→∅),[3.631]→[3.3820:3841](∅→∅),[3.3841]→[2.6148:6195](∅→∅),[2.6195]→[3.1226:1324](∅→∅),[3.1226]→[3.1226:1324](∅→∅),[3.1324]→[2.6196:6651](∅→∅),[2.6651]→[3.1324:1346](∅→∅),[3.1324]→[3.1324:1346](∅→∅),[3.1346]→[2.6652:6673](∅→∅),[2.6673]→[3.2905:2950](∅→∅),[3.1346]→[3.2905:2950](∅→∅),[3.2950]→[2.6674:6775](∅→∅),[2.6775]→[3.3000:3074](∅→∅),[3.3000]→[3.3000:3074](∅→∅),[3.3074]→[2.6776:6886](∅→∅),[3.3074]→[3.1346:1364](∅→∅),[2.6886]→[3.1346:1364](∅→∅),[3.1346]→[3.1346:1364](∅→∅),[3.1444]→[3.8797:8811](∅→∅),[3.8797]→[3.8797:8811](∅→∅)

if (defined $inputInfo->{$argName}) {
# The argument name matches an input.
$$usedInputs{$argName} = $inputInfo->{$argName};
if (defined $inputInfo->{$argName}->{storePath}) {
# !!! escaping
$extraArgs .= " --arg $argName '{path = builtins.toPath " . $inputInfo->{$argName}->{storePath} . ";}'";
} elsif (defined $inputInfo->{$argName}->{value}) {
$extraArgs .= " --argstr $argName '" . $inputInfo->{$argName}->{value} . "'";
}
}

else {
(my $prevBuild) = $db->resultset('Builds')->search(
{project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
{order_by => "timestamp DESC", rows => 1});

my $storePath;

if (!defined $prevBuild) {
# !!! reschedule?
die "missing input `$argName'";
}

# The argument name matches a previously built
# job in this jobset. Pick the most recent
# build. !!! refine the selection criteria:
# e.g., most recent successful build.
if (!isValidPath($prevBuild->outpath)) {
die "input path " . $prevBuild->outpath . " has been garbage-collected";
}

$$usedInputs{$argName} =
{ type => "build"
, storePath => $prevBuild->outpath
, id => $prevBuild->id
};

$extraArgs .= " --arg $argName '{path = builtins.toPath " . $prevBuild->outpath . ";}'";
}
}

[3.8609]

[3.8811]

push @argsNeeded, $argName;
}
edit in src/scheduler.pl at line 275

[3.8821]→[3.8821:8866](∅→∅),[3.8866]→[2.6887:6919](∅→∅),[3.1475]→[3.8866:9062](∅→∅),[2.6919]→[3.8866:9062](∅→∅),[3.8866]→[3.8866:9062](∅→∅)

# Instantiate the store derivation.
print $extraArgs, "\n";
my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs`
or die "cannot evaluate the Nix expression containing the job definitions: $?";
chomp $drvPath;
replacement in src/scheduler.pl at line 276

[3.9063]→[3.9063:9539](∅→∅),[3.9539]→[2.6920:6982](∅→∅),[2.6982]→[3.9603:9806](∅→∅),[3.9603]→[3.9603:9806](∅→∅),[3.9806]→[2.6983:7080](∅→∅)

# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs`
or die "cannot get information about the job: $?";

my $info = XMLin($infoXml, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";

my $job = $info->{item};
die if !defined $job || $job->{attrPath} ne $jobName;

my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
die unless $job->{drvPath} eq $drvPath;
my $outPath = $job->{outPath};

buildJob($project, $jobset, $jobName, $drvPath, $outPath, $usedInputs, $job->{system});

[3.9063]

[2.7080]

checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, "", \@argsNeeded, 0);
edit in src/scheduler.pl at line 280

[2.7086]→[2.7086:7289](∅→∅),[2.7289]→[3.9806:9807](∅→∅),[3.9806]→[3.9806:9807](∅→∅),[3.9807]→[2.7290:7600](∅→∅),[3.3940]→[3.9874:9880](∅→∅),[2.7600]→[3.9874:9880](∅→∅),[3.9874]→[3.9874:9880](∅→∅),[3.9880]→[2.7601:7802](∅→∅)

};

sub checkJobSetAlts {
my ($project, $jobset, $inputs, $n, $inputInfo) = @_;

if ($n >= scalar @{$inputs}) {
checkJobSetInstance($project, $jobset, $inputInfo);
return;
}

my $input = @{$inputs}[$n];

foreach my $alt ($input->jobsetinputalts) {
print " INPUT ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo); # !!! caching
checkJobSetAlts($project, $jobset, $inputs, $n + 1, $inputInfo);
}
};

sub checkJobSet {
my ($project, $jobset) = @_;
my $inputInfo = {};
my @jobsetinputs = $jobset->jobsetinputs;
checkJobSetAlts($project, $jobset, \@jobsetinputs, 0, $inputInfo);