hydra-eval-jobs: Use JSON instead of XML

[?]

Sep 29, 2014, 10:20 PM

IMQRX4MPANNOXDP6THT2FA4576CDFY44LGNCS74BHV6DWLOCXQOQC

Dependencies

[2] 5GPK54IV Log segfaults from the evaluator
[3] JTHWA6AM Rename aggregate members to constituents
[4] SS4TZXNU Distinguish between permanent evaluation errors and transient input errors
[5] KJNFONHP Bump default silent timeout to 2 hours
[6] L4LBF7UF Handle derivations without a system attribute
[7] YNGIYQRF hydra-eval-jobs: Don't keep track of used inputs
[8] 3PNG7NIB Remove trailing whitespace
[9] QE253KG4 * Option to show the tail of a log.
[10] 77VF5TC6 * For ease of use during debugging, don't require --gc-roots-dir.
[11] E7M2WP7A Remove unused Jobs columns
[12] YGRLM2SK * Export all relevant info about the derivation.
[13] 4LWGZL33
[14] YFPZ46YK * hydra: added variant of build input type, 'build output (same system)' to allow better continous integration in one jobset for multiple system. it makes sure that the system of the build that is passed as input for a job has the same system as the job.
[15] D7GTR6PP Remove wacky "sysbuild" filtering
[16] ZTQEU5QS Hydra: Add support for maxSilent meta attribute (also already added timeout, but not implemented the actual timeout for the build yet)
[17] 7YCFGMZB * Report evaluation errors per job, and don't bail out if a job fails
[18] PHNLYPKB Call buildFinished when a cached build is added
[19] P76HTFOD Revert "Remove wacky "sysbuild" filtering"
[20] A63IHCMX * Register GC roots properly.
[21] FGUL3HAZ hydra-eval-jobs: Don't go into an infinite recursion
[22] WUMCRXFX Don't ignore assertion failures in getDerivation()
[23] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
[24] JAH3UPWA Support revision control systems via plugins
[25] DPYJFBXU hydra-eval-jobs: Support meta.license being a list
[26] CQTN62OH Die tabs die
[27] SHZLOM5M * eval-jobs -> hydra_eval_jobs.
[28] HVXL2XUZ
[29] PMNWRTGJ Add multiple output support
[30] P45ANG5B Fix build
[31] CHQEG6WY Hydra/29: Added timeout to svn/git checkouts, job evaluation
[32] 5MNUNZWR * Store meta.maintainers.
[33] FYO6NECE hydra
[34] AEFNBIIW * Merge the GC branch.
[35] AEKIREIH * Updated hydra_eval_jobs for the new evaluator.
[36] 4N5APGRG * Start of a helper tool to evaluate job expressions efficiently.
[37] PXUCXYZI * Pass `-j 1' to hydra_eval_jobs to ensure that it can make progress
[38] POPU2ATH * hydra_scheduler: use eval-jobs.
[39] 5DYPEBSI hydra-eval-jobs: Use function argument default values
[40] RFE6T5LG * Store jobset evaluations in the database explicitly. This includes
[41] CGHMXHOL Remove dead code
[42] PIMGMGAF Rename hydra_eval_jobs to hydra-eval-jobs
[43] GA6XIXAN * Show the input bindings in evaluation error messages.
[44] RXVJFQ5A Evaluator cleanups
[45] R42G3IBQ Disallow multiple jobs with the same name
[46] WJTP6VZI Fix building against the latest Nix
[47] JOYONH2K Prevent multiple builds with the same (job, outPath) tuple from being added
[48] HJYRK37A * Revert r27114, it seems to break stuff.
[49] KD5237CU * eval-jobs now efficiently evaluates all Hydra jobs from a
[50] A43SLRSH Fix handling of IPC::Run::run exit status
[51] 7GKAIP3V Fix build and handling of string inputs starting with a dash
[52] INNOEHO6 * Fix getBuildLog for bzip2'd files.
[53] 5X6FHW3S hydra-eval-jobs: Fix building against the latest unstable Nix
[54] QMW24O5S Add support for Guile & Guix.
[55] AKRVETP5 Handle UTF-8 characters in eval error messages
[56] FTPCV25M Store aggregate members in the database
[57] 7BRTFGVY Really fix hydra-eval-jobs
[58] SRBU2RAE Warn against multiple jobs with the same name
[59] 3XTHEUMP * Implemented the clone feature.
[60] VTNP5KDO * Don't catch all errors. Some errors mess up the connection with the
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

replacement in src/c/hydra-eval-jobs.cc at line 11

[8.45]→[8.45:70](∅→∅)

#include "xml-writer.hh"

[8.45]

[8.0]

#include "value-to-json.hh"
replacement in src/c/hydra-eval-jobs.cc at line 26

[8.161]→[8.161:218](∅→∅)

static void findJobs(EvalState & state, XMLWriter & doc,

[8.161]

[7.0]

static void findJobs(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 30

[8.98]→[8.98:157](∅→∅)

static void tryJobAlts(EvalState & state, XMLWriter & doc,

[8.98]

[7.69]

static void tryJobAlts(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 41

[8.676]→[7.139:192](∅→∅)

findJobs(state, doc, argsLeft, v, attrPath);

[8.676]

[8.448]

findJobs(state, top, argsLeft, v, attrPath);
replacement in src/c/hydra-eval-jobs.cc at line 53

[8.177]→[7.193:274](∅→∅)

tryJobAlts(state, doc, argsLeft, attrPath, fun, next, last, actualArgs);

[8.177]

[8.268]

tryJobAlts(state, top, argsLeft, attrPath, fun, next, last, actualArgs);
replacement in src/c/hydra-eval-jobs.cc at line 66

[8.1166]→[7.275:358](∅→∅)

tryJobAlts(state, doc, argsLeft2, attrPath, fun, next, last, actualArgs2);

[8.1166]

[8.1260]

tryJobAlts(state, top, argsLeft2, attrPath, fun, next, last, actualArgs2);
replacement in src/c/hydra-eval-jobs.cc at line 96

[8.1574]→[8.313:377](∅→∅),[8.1413]→[8.313:377](∅→∅)

static void findJobsWrapped(EvalState & state, XMLWriter & doc,

[8.1574]

[7.359]

static void findJobsWrapped(EvalState & state, JSONObject & top,
edit in src/c/hydra-eval-jobs.cc at line 110

[8.78]→[8.681:712](∅→∅),[8.112]→[8.681:712](∅→∅),[8.1773]→[8.681:712](∅→∅),[8.681]→[8.681:712](∅→∅)

XMLAttrs xmlAttrs;
replacement in src/c/hydra-eval-jobs.cc at line 117

[6.124]→[8.6:94](∅→∅),[8.748]→[8.6:94](∅→∅),[8.94]→[8.113:158](∅→∅),[8.789]→[8.113:158](∅→∅),[8.158]→[8.742:1188](∅→∅),[8.562]→[8.0:1](∅→∅),[8.1188]→[8.0:1](∅→∅),[8.1131]→[8.0:1](∅→∅),[8.1]→[8.1189:1291](∅→∅),[8.1291]→[8.2:3](∅→∅),[8.1269]→[8.2:3](∅→∅),[8.3]→[8.1292:1374](∅→∅),[8.127]→[8.0:1](∅→∅),[8.1374]→[8.0:1](∅→∅),[8.1269]→[8.0:1](∅→∅),[8.1]→[5.0:85](∅→∅)

xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
xmlAttrs["system"] = drv.system;
xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath();
xmlAttrs["description"] = drv.queryMetaString("description");
xmlAttrs["longDescription"] = drv.queryMetaString("longDescription");
xmlAttrs["license"] = queryMetaStrings(state, drv, "license");
xmlAttrs["homepage"] = drv.queryMetaString("homepage");
xmlAttrs["maintainers"] = queryMetaStrings(state, drv, "maintainers");

xmlAttrs["schedulingPriority"] = int2String(drv.queryMetaInt("schedulingPriority", 100));

xmlAttrs["timeout"] = int2String(drv.queryMetaInt("timeout", 36000));

xmlAttrs["maxSilent"] = int2String(drv.queryMetaInt("maxSilent", 7200));

[6.124]

[8.259]

top.attr(attrPath);
JSONObject res(top.str);
res.attr("nixName", drv.name);
res.attr("system", drv.system);
res.attr("drvPath", drvPath = drv.queryDrvPath());
res.attr("description", drv.queryMetaString("description"));
res.attr("longDescription", drv.queryMetaString("longDescription"));
res.attr("license", queryMetaStrings(state, drv, "license"));
res.attr("homepage", drv.queryMetaString("homepage"));
res.attr("maintainers", queryMetaStrings(state, drv, "maintainers"));
res.attr("schedulingPriority", drv.queryMetaInt("schedulingPriority", 100));
res.attr("timeout", drv.queryMetaInt("timeout", 36000));
res.attr("maxSilent", drv.queryMetaInt("maxSilent", 7200));
replacement in src/c/hydra-eval-jobs.cc at line 145

[8.811]→[3.257:329](∅→∅)

xmlAttrs["constituents"] = concatStringsSep(" ", drvs);

[8.811]

[8.878]

res.attr("constituents", concatStringsSep(" ", drvs));
replacement in src/c/hydra-eval-jobs.cc at line 156

[8.425]→[8.1096:1148](∅→∅),[8.805]→[8.1096:1148](∅→∅),[8.1096]→[8.1096:1148](∅→∅),[8.1148]→[8.66:321](∅→∅)

XMLOpenElement _(doc, "job", xmlAttrs);

foreach (DrvInfo::Outputs::iterator, j, outputs) {
XMLAttrs attrs2;
attrs2["name"] = j->first;
attrs2["path"] = j->second;
doc.writeEmptyElement("output", attrs2);
}

[8.805]

[8.1148]

res.attr("outputs");
JSONObject res2(res.str);
for (auto & j : outputs)
res2.attr(j.first, j.second);
replacement in src/c/hydra-eval-jobs.cc at line 165

[8.178]→[7.427:489](∅→∅)

findJobs(state, doc, argsLeft, *i->value,

[8.178]

[8.250]

findJobs(state, top, argsLeft, *i->value,
replacement in src/c/hydra-eval-jobs.cc at line 173

[8.49]→[7.490:544](∅→∅)

tryJobAlts(state, doc, argsLeft, attrPath, v,

[8.49]

[8.2112]

tryJobAlts(state, top, argsLeft, attrPath, v,
replacement in src/c/hydra-eval-jobs.cc at line 188

[8.494]→[8.494:551](∅→∅)

static void findJobs(EvalState & state, XMLWriter & doc,

[8.494]

[7.545]

static void findJobs(EvalState & state, JSONObject & top,
replacement in src/c/hydra-eval-jobs.cc at line 192

[8.658]→[7.613:673](∅→∅)

findJobsWrapped(state, doc, argsLeft, v, attrPath);

[8.658]

[8.0]

findJobsWrapped(state, top, argsLeft, v, attrPath);
replacement in src/c/hydra-eval-jobs.cc at line 194

[8.30]→[8.754:907](∅→∅),[8.754]→[8.754:907](∅→∅)

XMLAttrs xmlAttrs;
xmlAttrs["location"] = attrPath;
xmlAttrs["msg"] = e.msg();
XMLOpenElement _(doc, "error", xmlAttrs);

[8.30]

[8.944]

top.attr(attrPath);
JSONObject res(top.str);
res.attr("error", e.msg());
top.str << "\n";
replacement in src/c/hydra-eval-jobs.cc at line 262

[8.1022]→[8.1985:2067](∅→∅),[8.2067]→[7.674:721](∅→∅)

XMLWriter doc(true, std::cout);
XMLOpenElement root(doc, "jobs");
findJobs(state, doc, autoArgs, v, "");

[8.1022]

[8.1022]

JSONObject json(std::cout);
findJobs(state, json, autoArgs, v, "");
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 6

[8.10]→[8.1272:1289](∅→∅),[8.1272]→[8.1272:1289](∅→∅)

use XML::Simple;

[8.10]

[8.1290]

use JSON;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 338

[8.414]→[8.838:906](∅→∅),[8.2529]→[8.838:906](∅→∅),[8.2918]→[8.838:906](∅→∅)

(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(10800,

[8.2529]

[8.3347]

(my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(10800,
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 340

[8.3465]→[2.0:178](∅→∅)

if ($res) {
die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? $stderr : "(no output)\n");
}

[8.3465]

[8.3181]

die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? $stderr : "(no output)\n")
if $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 346

[8.3211]→[8.3211:3251](∅→∅),[8.3251]→[8.4952:5051](∅→∅),[8.5051]→[8.3321:3393](∅→∅),[8.3321]→[8.3321:3393](∅→∅),[8.3393]→[8.0:34](∅→∅),[8.34]→[8.0:67](∅→∅),[8.3393]→[8.0:67](∅→∅),[8.67]→[8.35:431](∅→∅),[8.431]→[7.722:756](∅→∅),[7.756]→[8.360:401](∅→∅),[8.360]→[8.360:401](∅→∅),[8.303]→[8.11:140](∅→∅),[8.140]→[8.303:348](∅→∅),[8.303]→[8.303:348](∅→∅)

my $jobs = XMLin(
$jobsXml,
ForceArray => ['error', 'job', 'arg', 'output'],
KeyAttr => { output => "+name" },
SuppressEmpty => '')
or die "cannot parse XML output";

my %jobNames;
my $errors;
my @filteredJobs = ();
foreach my $job (@{$jobs->{job}}) {
# Ensure that there is only one job with the given
# name. FIXME: this check will become unnecessary after we
# remove support for multiple values per jobset input.
if (defined $jobNames{$job->{jobName}}) {
$errors .= "error: there are multiple jobs named ‘$job->{jobName}’\n\n";
next;
}
$jobNames{$job->{jobName}} = 1;
push @filteredJobs, $job;
}
$jobs->{job} = \@filteredJobs;

# Handle utf-8 characters in error messages. No idea why this works.
utf8::decode($_->{msg}) foreach @{$jobs->{error}};

return ($jobs, $nixExprInput, $errors);

[8.3211]

[8.3428]

return (decode_json($jobsJSON), $nixExprInput);
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 440

[8.7100]→[8.7100:7157](∅→∅)

my @outputNames = sort keys %{$buildInfo->{output}};

[8.7100]

[8.7157]

my @outputNames = sort keys %{$buildInfo->{outputs}};
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 447

[8.7402]→[8.7402:7478](∅→∅)

my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path};

[8.7402]

[8.7478]

my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 452

[8.3725]→[8.3725:3860](∅→∅)

my $priority = 100;
$priority = int($buildInfo->{schedulingPriority})
if $buildInfo->{schedulingPriority} =~ /^\d+$/;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 503

[8.8455]→[8.8455:8517](∅→∅)

my $path = $buildInfo->{output}->{$name}->{path};

[8.8455]

[8.8517]

my $path = $buildInfo->{outputs}->{$name};
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 545

[8.5872]→[8.4551:4587](∅→∅)

, priority => $priority

[8.5872]

[8.4587]

, priority => $buildInfo->{schedulingPriority}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 551

[8.9047]→[8.9047:9144](∅→∅)

$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} })

[8.9047]

[8.9144]

$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
replacement in src/script/hydra-evaluator at line 147

[8.5549]→[8.349:465](∅→∅)

my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);

[8.5549]

[8.5550]

my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
edit in src/script/hydra-evaluator at line 149

[8.5575]

[8.547]

$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
replacement in src/script/hydra-evaluator at line 165

[8.2297]→[8.5655:5750](∅→∅),[8.4505]→[8.5655:5750](∅→∅),[8.5655]→[8.5655:5750](∅→∅)

foreach my $job (permute @{$jobs->{job}}) {
next if $job->{jobName} eq "";

[8.4505]

[8.2298]

foreach my $job (permute(values %{$jobs})) {
next if defined $job->{error};
replacement in src/script/hydra-evaluator at line 249

[8.773]→[8.773:817](∅→∅)

foreach my $error (@{$jobs->{error}}) {

[8.2840]

[8.152]

my $msg = "";
foreach my $job (values %{$jobs}) {
next unless defined $job->{error};
replacement in src/script/hydra-evaluator at line 253

[8.168]→[8.168:262](∅→∅),[8.262]→[4.1228:1272](∅→∅)

($error->{location} ne "" ? "in job ‘$error->{location}’" : "at top-level") .
":\n" . $error->{msg} . "\n\n";

[8.168]

[8.4988]

($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
":\n" . $job->{error} . "\n\n";