Output evaluation errors without crashing if aggregate job is broken.

Dependencies

• [2] W7OCNTLF Fix build with latest Nix
• [3] HWGFAF2B queue runner: test notifications
• [4] VA3Z62HQ Revert "Fix unhelpful error messages in aggregate jobs."
• [5] 4DHGJ2FL Fix build
• [6] 6NUK6AZA Fix build
• [7] NQPGIRXX Revert "hydra-eval-jobs -> nix eval-hydra-jobs"
• [8] VUYJ47EV hydra-eval-jobs: Parallelize
• [9] D4G756UK hydra-eval-jobs: Identify unexpected errors in handling aggregate jobs
• [*] 2JJP7673 tests: move to t, allow `yath test` from root

Change contents

• edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 1
  [5.530]→[5.531:546](∅→∅)
  − #include <map>
• edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 3
  [2.18]

  [5.623]
  + #include <optional>

  + #include <unordered_map>
• edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 445
  [5.12789]

  [5.12789]
  +

  + std::unordered_map<std::string, std::string> brokenJobs;

  + auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](

  + const std::string & childJobName) -> std::optional<nlohmann::json> {

  + auto childJob = state->jobs.find(childJobName);

  + if (childJob == state->jobs.end()) {

  + printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);

  + brokenJobs[childJobName] = "does not exist";

  + return std::nullopt;

  + }

  + if (childJob->find("error") != childJob->end()) {

  + std::string error = (*childJob)["error"];

  + printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);

  + brokenJobs[childJobName] = error;

  + return std::nullopt;

  + }

  + return *childJob;

  + };
• replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 466
  [4.87]→[4.87:309](∅→∅)
  − auto job2 = state->jobs.find(jobName2);

  − if (job2 == state->jobs.end())

  − throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
  [4.87]

  [4.309]
  + auto job2 = getNonBrokenJobOrRecordError(jobName2);

  + if (!job2) {

  + continue;

  + }
• replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 478
  [4.670]→[4.670:892](∅→∅)
  − auto job2 = state->jobs.find(jobName2);

  − if (job2 == state->jobs.end())

  − throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
  [4.670]

  [4.892]
  + auto job2 = getNonBrokenJobOrRecordError(jobName2);

  + if (!job2) {

  + continue;

  + }
• replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 488
  [5.2181]→[4.1230:1854](∅→∅)
  − std::string drvName(drvPath.name());

  − assert(hasSuffix(drvName, drvExtension));

  − drvName.resize(drvName.size() - drvExtension.size());

  − auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));

  − auto outPath = store->makeOutputPath("out", h, drvName);

  − drv.env["out"] = store->printStorePath(outPath);

  − drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });

  − auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
  [5.2181]

  [5.14561]
  + if (brokenJobs.empty()) {

  + std::string drvName(drvPath.name());

  + assert(hasSuffix(drvName, drvExtension));

  + drvName.resize(drvName.size() - drvExtension.size());

  + auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));

  + auto outPath = store->makeOutputPath("out", h, drvName);

  + drv.env["out"] = store->printStorePath(outPath);

  + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });

  + auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
• replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 498
  [5.14562]→[4.1855:1963](∅→∅)
  − debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
  [5.14562]

  [5.14647]
  + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);
• replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 500
  [5.14648]→[4.1964:2081](∅→∅)
  − job["drvPath"] = newDrvPath;

  − job["outputs"]["out"] = store->printStorePath(outPath);
  [5.14648]

  [5.14765]
  + job["drvPath"] = newDrvPath;

  + job["outputs"]["out"] = store->printStorePath(outPath);

  + }
• edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 506
  [5.14824]

  [5.9592]
  +

  + if (!brokenJobs.empty()) {

  + std::stringstream ss;

  + for (const auto& [jobName, error] : brokenJobs) {

  + ss << jobName << ": " << error << "\n";

  + }

  + job["error"] = ss.str();

  + }
• file addition: broken-constituent.nix (----------)
  [11.1263]
  + with import ./config.nix;

  + {

  + broken = mkDerivation {

  + name = "broken";

  + _hydraAggregate = true;

  + constituents = [

  + "does-not-exist"

  + "does-not-evaluate"

  + ];

  + builder = ./fail.sh;

  + };

  +

  + # does-not-exist doesn't exist.

  +

  + does-not-evaluate = assert false; {};

  + }
• file addition: constituents.t (----------)
  [3.321]
  + use feature 'unicode_strings';

  + use strict;

  + use warnings;

  + use Setup;

  +

  + my %ctx = test_init();

  +

  + require Hydra::Schema;

  + require Hydra::Model::DB;

  +

  + use Test2::V0;

  +

  + my $db = Hydra::Model::DB->new;

  + hydra_setup($db);

  +

  + my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});

  +

  + my $jobset = createBaseJobset("broken-constituent", "broken-constituent.nix", $ctx{jobsdir});

  +

  + ok(evalSucceeds($jobset), "Evaluating jobs/broken-constituent.nix should exit with return code 0");

  + is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix should not queue any builds");

  +

  + like(

  + $jobset->errormsg,

  + qr/^does-not-exist: does not exist$/m,

  + "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");

  + like(

  + $jobset->errormsg,

  + qr/^does-not-evaluate: error: assertion 'false' failed$/m,

  + "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");

  +

  + done_testing;