hydra-eval-jobs: Use JSON instead of XML

[?]

Sep 29, 2014, 10:20 PM

IMQRX4MPANNOXDP6THT2FA4576CDFY44LGNCS74BHV6DWLOCXQOQC

Dependencies

[2] 5GPK54IV Log segfaults from the evaluator
[3] JTHWA6AM Rename aggregate members to constituents
[4] SS4TZXNU Distinguish between permanent evaluation errors and transient input errors
[5] KJNFONHP Bump default silent timeout to 2 hours
[6] L4LBF7UF Handle derivations without a system attribute
[7] YNGIYQRF hydra-eval-jobs: Don't keep track of used inputs
[8] YFPZ46YK * hydra: added variant of build input type, 'build output (same system)' to allow better continous integration in one jobset for multiple system. it makes sure that the system of the build that is passed as input for a job has the same system as the job.
[9] POPU2ATH * hydra_scheduler: use eval-jobs.
[10] 3PNG7NIB Remove trailing whitespace
[11] FGUL3HAZ hydra-eval-jobs: Don't go into an infinite recursion
[12] 5DYPEBSI hydra-eval-jobs: Use function argument default values
[13] HVXL2XUZ
[14] E7M2WP7A Remove unused Jobs columns
[15] CGHMXHOL Remove dead code
[16] A43SLRSH Fix handling of IPC::Run::run exit status
[17] AEKIREIH * Updated hydra_eval_jobs for the new evaluator.
[18] 7YCFGMZB * Report evaluation errors per job, and don't bail out if a job fails
[19] SRBU2RAE Warn against multiple jobs with the same name
[20] 4LWGZL33
[21] 4N5APGRG * Start of a helper tool to evaluate job expressions efficiently.
[22] JOYONH2K Prevent multiple builds with the same (job, outPath) tuple from being added
[23] 5MNUNZWR * Store meta.maintainers.
[24] D7GTR6PP Remove wacky "sysbuild" filtering
[25] INNOEHO6 * Fix getBuildLog for bzip2'd files.
[26] R42G3IBQ Disallow multiple jobs with the same name
[27] FYO6NECE hydra
[28] YGRLM2SK * Export all relevant info about the derivation.
[29] KD5237CU * eval-jobs now efficiently evaluates all Hydra jobs from a
[30] VTNP5KDO * Don't catch all errors. Some errors mess up the connection with the
[31] WJTP6VZI Fix building against the latest Nix
[32] 7BRTFGVY Really fix hydra-eval-jobs
[33] SHZLOM5M * eval-jobs -> hydra_eval_jobs.
[34] JAH3UPWA Support revision control systems via plugins
[35] QMW24O5S Add support for Guile & Guix.
[36] A63IHCMX * Register GC roots properly.
[37] P45ANG5B Fix build
[38] CHQEG6WY Hydra/29: Added timeout to svn/git checkouts, job evaluation
[39] RXVJFQ5A Evaluator cleanups
[40] 3XTHEUMP * Implemented the clone feature.
[41] ZTQEU5QS Hydra: Add support for maxSilent meta attribute (also already added timeout, but not implemented the actual timeout for the build yet)
[42] RFE6T5LG * Store jobset evaluations in the database explicitly. This includes
[43] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
[44] 77VF5TC6 * For ease of use during debugging, don't require --gc-roots-dir.
[45] GA6XIXAN * Show the input bindings in evaluation error messages.
[46] WUMCRXFX Don't ignore assertion failures in getDerivation()
[47] PMNWRTGJ Add multiple output support
[48] FTPCV25M Store aggregate members in the database
[49] P76HTFOD Revert "Remove wacky "sysbuild" filtering"
[50] PIMGMGAF Rename hydra_eval_jobs to hydra-eval-jobs
[51] AEFNBIIW * Merge the GC branch.
[52] PXUCXYZI * Pass `-j 1' to hydra_eval_jobs to ensure that it can make progress
[53] PHNLYPKB Call buildFinished when a cached build is added
[54] 5X6FHW3S hydra-eval-jobs: Fix building against the latest unstable Nix
[55] AKRVETP5 Handle UTF-8 characters in eval error messages
[56] DPYJFBXU hydra-eval-jobs: Support meta.license being a list
[57] QE253KG4 * Option to show the tail of a log.
[58] 7GKAIP3V Fix build and handling of string inputs starting with a dash
[59] HJYRK37A * Revert r27114, it seems to break stuff.
[60] CQTN62OH Die tabs die
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

replacement in src/c/hydra-eval-jobs.cc at line 11

[8.45]→[8.45:70](∅→∅)

#include "xml-writer.hh"

[8.45]

[8.0]

#include "value-to-json.hh"
replacement in src/c/hydra-eval-jobs.cc at line 26

[8.161]→[8.161:218](∅→∅)

static void findJobs(EvalState & state, XMLWriter & doc,

[8.161]

[7.0]

static void findJobs(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 30

[8.98]→[8.98:157](∅→∅)

static void tryJobAlts(EvalState & state, XMLWriter & doc,

[8.98]

[7.69]

static void tryJobAlts(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 41

[8.676]→[7.139:192](∅→∅)

findJobs(state, doc, argsLeft, v, attrPath);

[8.676]

[8.448]

findJobs(state, top, argsLeft, v, attrPath);
replacement in src/c/hydra-eval-jobs.cc at line 53

[8.177]→[7.193:274](∅→∅)

tryJobAlts(state, doc, argsLeft, attrPath, fun, next, last, actualArgs);

[8.177]

[8.268]

tryJobAlts(state, top, argsLeft, attrPath, fun, next, last, actualArgs);
replacement in src/c/hydra-eval-jobs.cc at line 66

[8.1166]→[7.275:358](∅→∅)

tryJobAlts(state, doc, argsLeft2, attrPath, fun, next, last, actualArgs2);

[8.1166]

[8.1260]

tryJobAlts(state, top, argsLeft2, attrPath, fun, next, last, actualArgs2);
replacement in src/c/hydra-eval-jobs.cc at line 96

[8.1574]→[8.313:377](∅→∅),[8.1413]→[8.313:377](∅→∅)

static void findJobsWrapped(EvalState & state, XMLWriter & doc,

[8.1574]

[7.359]

static void findJobsWrapped(EvalState & state, JSONObject & top,
edit in src/c/hydra-eval-jobs.cc at line 110

[8.78]→[8.681:712](∅→∅),[8.112]→[8.681:712](∅→∅),[8.1773]→[8.681:712](∅→∅),[8.681]→[8.681:712](∅→∅)

XMLAttrs xmlAttrs;
replacement in src/c/hydra-eval-jobs.cc at line 117

[6.124]→[8.6:94](∅→∅),[8.748]→[8.6:94](∅→∅),[8.94]→[8.113:158](∅→∅),[8.789]→[8.113:158](∅→∅),[8.158]→[8.742:1188](∅→∅),[8.562]→[8.0:1](∅→∅),[8.1188]→[8.0:1](∅→∅),[8.1131]→[8.0:1](∅→∅),[8.1]→[8.1189:1291](∅→∅),[8.1291]→[8.2:3](∅→∅),[8.1269]→[8.2:3](∅→∅),[8.3]→[8.1292:1374](∅→∅),[8.127]→[8.0:1](∅→∅),[8.1374]→[8.0:1](∅→∅),[8.1269]→[8.0:1](∅→∅),[8.1]→[5.0:85](∅→∅)

xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
xmlAttrs["system"] = drv.system;
xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath();
xmlAttrs["description"] = drv.queryMetaString("description");
xmlAttrs["longDescription"] = drv.queryMetaString("longDescription");
xmlAttrs["license"] = queryMetaStrings(state, drv, "license");
xmlAttrs["homepage"] = drv.queryMetaString("homepage");
xmlAttrs["maintainers"] = queryMetaStrings(state, drv, "maintainers");

xmlAttrs["schedulingPriority"] = int2String(drv.queryMetaInt("schedulingPriority", 100));

xmlAttrs["timeout"] = int2String(drv.queryMetaInt("timeout", 36000));

xmlAttrs["maxSilent"] = int2String(drv.queryMetaInt("maxSilent", 7200));

[6.124]

[8.259]

top.attr(attrPath);
JSONObject res(top.str);
res.attr("nixName", drv.name);
res.attr("system", drv.system);
res.attr("drvPath", drvPath = drv.queryDrvPath());
res.attr("description", drv.queryMetaString("description"));
res.attr("longDescription", drv.queryMetaString("longDescription"));
res.attr("license", queryMetaStrings(state, drv, "license"));
res.attr("homepage", drv.queryMetaString("homepage"));
res.attr("maintainers", queryMetaStrings(state, drv, "maintainers"));
res.attr("schedulingPriority", drv.queryMetaInt("schedulingPriority", 100));
res.attr("timeout", drv.queryMetaInt("timeout", 36000));
res.attr("maxSilent", drv.queryMetaInt("maxSilent", 7200));
replacement in src/c/hydra-eval-jobs.cc at line 145

[8.811]→[3.257:329](∅→∅)

xmlAttrs["constituents"] = concatStringsSep(" ", drvs);

[8.811]

[8.878]

res.attr("constituents", concatStringsSep(" ", drvs));
replacement in src/c/hydra-eval-jobs.cc at line 156

[8.425]→[8.1096:1148](∅→∅),[8.805]→[8.1096:1148](∅→∅),[8.1096]→[8.1096:1148](∅→∅),[8.1148]→[8.66:321](∅→∅)

XMLOpenElement _(doc, "job", xmlAttrs);

foreach (DrvInfo::Outputs::iterator, j, outputs) {
XMLAttrs attrs2;
attrs2["name"] = j->first;
attrs2["path"] = j->second;
doc.writeEmptyElement("output", attrs2);
}

[8.805]

[8.1148]

res.attr("outputs");
JSONObject res2(res.str);
for (auto & j : outputs)
res2.attr(j.first, j.second);
replacement in src/c/hydra-eval-jobs.cc at line 165

[8.178]→[7.427:489](∅→∅)

findJobs(state, doc, argsLeft, *i->value,

[8.178]

[8.250]

findJobs(state, top, argsLeft, *i->value,
replacement in src/c/hydra-eval-jobs.cc at line 173

[8.49]→[7.490:544](∅→∅)

tryJobAlts(state, doc, argsLeft, attrPath, v,

[8.49]

[8.2112]

tryJobAlts(state, top, argsLeft, attrPath, v,
replacement in src/c/hydra-eval-jobs.cc at line 188

[8.494]→[8.494:551](∅→∅)

static void findJobs(EvalState & state, XMLWriter & doc,

[8.494]

[7.545]

static void findJobs(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 192

[8.658]→[7.613:673](∅→∅)

findJobsWrapped(state, doc, argsLeft, v, attrPath);

[8.658]

[8.0]

findJobsWrapped(state, top, argsLeft, v, attrPath);
replacement in src/c/hydra-eval-jobs.cc at line 194

[8.30]→[8.754:907](∅→∅),[8.754]→[8.754:907](∅→∅)

XMLAttrs xmlAttrs;
xmlAttrs["location"] = attrPath;
xmlAttrs["msg"] = e.msg();
XMLOpenElement _(doc, "error", xmlAttrs);

[8.30]

[8.944]

top.attr(attrPath);
JSONObject res(top.str);
res.attr("error", e.msg());
top.str << "\n";
replacement in src/c/hydra-eval-jobs.cc at line 262

[8.1022]→[8.1985:2067](∅→∅),[8.2067]→[7.674:721](∅→∅)

XMLWriter doc(true, std::cout);
XMLOpenElement root(doc, "jobs");
findJobs(state, doc, autoArgs, v, "");

[8.1022]

[8.1022]

JSONObject json(std::cout);
findJobs(state, json, autoArgs, v, "");
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 6

[8.10]→[8.1272:1289](∅→∅),[8.1272]→[8.1272:1289](∅→∅)

use XML::Simple;

[8.10]

[8.1290]

use JSON;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 338

[8.414]→[8.838:906](∅→∅),[8.2529]→[8.838:906](∅→∅),[8.2918]→[8.838:906](∅→∅)

(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(10800,

[8.2529]

[8.3347]

(my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(10800,
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 340

[8.3465]→[2.0:178](∅→∅)

if ($res) {
die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? $stderr : "(no output)\n");
}

[8.3465]

[8.3181]

die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? $stderr : "(no output)\n")
if $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 346

[8.3211]→[8.3211:3251](∅→∅),[8.3251]→[8.4952:5051](∅→∅),[8.5051]→[8.3321:3393](∅→∅),[8.3321]→[8.3321:3393](∅→∅),[8.3393]→[8.0:34](∅→∅),[8.34]→[8.0:67](∅→∅),[8.3393]→[8.0:67](∅→∅),[8.67]→[8.35:431](∅→∅),[8.431]→[7.722:756](∅→∅),[7.756]→[8.360:401](∅→∅),[8.360]→[8.360:401](∅→∅),[8.303]→[8.11:140](∅→∅),[8.140]→[8.303:348](∅→∅),[8.303]→[8.303:348](∅→∅)

my $jobs = XMLin(
$jobsXml,
ForceArray => ['error', 'job', 'arg', 'output'],
KeyAttr => { output => "+name" },
SuppressEmpty => '')
or die "cannot parse XML output";

my %jobNames;
my $errors;
my @filteredJobs = ();
foreach my $job (@{$jobs->{job}}) {
# Ensure that there is only one job with the given
# name. FIXME: this check will become unnecessary after we
# remove support for multiple values per jobset input.
if (defined $jobNames{$job->{jobName}}) {
$errors .= "error: there are multiple jobs named ‘$job->{jobName}’\n\n";
next;
}
$jobNames{$job->{jobName}} = 1;
push @filteredJobs, $job;
}
$jobs->{job} = \@filteredJobs;

# Handle utf-8 characters in error messages. No idea why this works.
utf8::decode($_->{msg}) foreach @{$jobs->{error}};

return ($jobs, $nixExprInput, $errors);

[8.3211]

[8.3428]

return (decode_json($jobsJSON), $nixExprInput);
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 440

[8.7100]→[8.7100:7157](∅→∅)

my @outputNames = sort keys %{$buildInfo->{output}};

[8.7100]

[8.7157]

my @outputNames = sort keys %{$buildInfo->{outputs}};
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 447

[8.7402]→[8.7402:7478](∅→∅)

my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path};

[8.7402]

[8.7478]

my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 452

[8.3725]→[8.3725:3860](∅→∅)

my $priority = 100;
$priority = int($buildInfo->{schedulingPriority})
if $buildInfo->{schedulingPriority} =~ /^\d+$/;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 503

[8.8455]→[8.8455:8517](∅→∅)

my $path = $buildInfo->{output}->{$name}->{path};

[8.8455]

[8.8517]

my $path = $buildInfo->{outputs}->{$name};
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 545

[8.5872]→[8.4551:4587](∅→∅)

, priority => $priority

[8.5872]

[8.4587]

, priority => $buildInfo->{schedulingPriority}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 551

[8.9047]→[8.9047:9144](∅→∅)

$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} })

[8.9047]

[8.9144]

$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
replacement in src/script/hydra-evaluator at line 147

[8.5549]→[8.349:465](∅→∅)

my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);

[8.5549]

[8.5550]

my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
edit in src/script/hydra-evaluator at line 149

[8.5575]

[8.547]

$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
replacement in src/script/hydra-evaluator at line 165

[8.2297]→[8.5655:5750](∅→∅),[8.4505]→[8.5655:5750](∅→∅),[8.5655]→[8.5655:5750](∅→∅)

foreach my $job (permute @{$jobs->{job}}) {
next if $job->{jobName} eq "";

[8.4505]

[8.2298]

foreach my $job (permute(values %{$jobs})) {
next if defined $job->{error};
replacement in src/script/hydra-evaluator at line 249

[8.773]→[8.773:817](∅→∅)

foreach my $error (@{$jobs->{error}}) {

[8.2840]

[8.152]

my $msg = "";
foreach my $job (values %{$jobs}) {
next unless defined $job->{error};
replacement in src/script/hydra-evaluator at line 253

[8.168]→[8.168:262](∅→∅),[8.262]→[4.1228:1272](∅→∅)

($error->{location} ne "" ? "in job ‘$error->{location}’" : "at top-level") .
":\n" . $error->{msg} . "\n\n";

[8.168]

[8.4988]

($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
":\n" . $job->{error} . "\n\n";