grahamc/hydra2 - Change GOVMQVEDNIXFRAIBG5T2HGYSSHMFH542W5H67PFVG74L5T2JABIQC

Output evaluation errors without crashing if aggregate job is broken.

At the moment, aggregate jobs can easily break and cause the entire evaluation to fail, which is not ideal. For Nixpkgs, we do have some important aggregate jobs (like tested), but for debugging and building purposes it's still useful to get a partial result even if the channel won't actually advance.

This commit changes the behaviour of hydra-eval-jobs such that it aggregates any errors found during the construction of an aggregate, and will instead annotate the job with the evaluation failure such that it shows up in a "cleaner" way.

There are really two types of failure that we care about: one is where the attribute just ends up missing altogether in the final output, and also where the attribute is in the output but fails to evaluate. Both are handled here.

Note that this does mean that the same error message may be output multiple times, but this aids debuggability because it'll be much clearer what's blocking the job from being created.

Created by Luke Granger-Brown on September 22, 2021

GOVMQVEDNIXFRAIBG5T2HGYSSHMFH542W5H67PFVG74L5T2JABIQC

Dependencies

In channels

main

Change contents

Deletion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 1 [5.530]
B:BD[5.530] → [5.531:546]
```
#include <map>
```
Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 3 [5.530]
[2.18]
[5.623]
```
#include <optional>
#include <unordered_map>
```

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 445 [5.530]

[6.12789]


            std::unordered_map<std::string, std::string> brokenJobs;
            auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](
                    const std::string & childJobName) -> std::optional<nlohmann::json> {
                auto childJob = state->jobs.find(childJobName);
                if (childJob == state->jobs.end()) {
                    printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
                    brokenJobs[childJobName] = "does not exist";
                    return std::nullopt;
                }
                if (childJob->find("error") != childJob->end()) {
                    std::string error = (*childJob)["error"];
                    printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);
                    brokenJobs[childJobName] = error;
                    return std::nullopt;
                }
                return *childJob;
            };

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 466 [5.530]

B:BD[4.87] → [4.87:309]

                    auto job2 = state->jobs.find(jobName2);
                    if (job2 == state->jobs.end())
                        throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);

[4.87]

[4.309]

                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
                    if (!job2) {
                        continue;
                    }

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 478 [5.530]

B:BD[4.670] → [4.670:892]

                    auto job2 = state->jobs.find(jobName2);
                    if (job2 == state->jobs.end())
                        throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);

[4.670]

[4.892]

                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
                    if (!job2) {
                        continue;
                    }

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 488 [5.530]

B:BD[7.2181] → [4.1230:1854]

                std::string drvName(drvPath.name());
                assert(hasSuffix(drvName, drvExtension));
                drvName.resize(drvName.size() - drvExtension.size());
                auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));
                auto outPath = store->makeOutputPath("out", h, drvName);
                drv.env["out"] = store->printStorePath(outPath);
                drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
                auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));

[7.2181]

[6.14561]

                if (brokenJobs.empty()) {
                    std::string drvName(drvPath.name());
                    assert(hasSuffix(drvName, drvExtension));
                    drvName.resize(drvName.size() - drvExtension.size());
                    auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));
                    auto outPath = store->makeOutputPath("out", h, drvName);
                    drv.env["out"] = store->printStorePath(outPath);
                    drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
                    auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 498 [5.530]

B:BD[6.14562] → [4.1855:1963]

                debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);

[6.14562]

[6.14647]

                    debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 500 [5.530]

B:BD[6.14648] → [4.1964:2081]

                job["drvPath"] = newDrvPath;
                job["outputs"]["out"] = store->printStorePath(outPath);

[6.14648]

[6.14765]

                    job["drvPath"] = newDrvPath;
                    job["outputs"]["out"] = store->printStorePath(outPath);
                }

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 506 [5.530]

[6.14824]

[5.9592]


            if (!brokenJobs.empty()) {
                std::stringstream ss;
                for (const auto& [jobName, error] : brokenJobs) {
                    ss << jobName << ": " << error << "\n";
                }
                job["error"] = ss.str();
            }

File addition: broken-constituent.nix (----------)

[8.1263]

with import ./config.nix;
{
  broken = mkDerivation {
    name = "broken";
    _hydraAggregate = true;
    constituents = [
      "does-not-exist"
      "does-not-evaluate"
    ];
    builder = ./fail.sh;
  };
  # does-not-exist doesn't exist.
  does-not-evaluate = assert false; {};
}

File addition: constituents.t (----------)

[3.321]

use feature 'unicode_strings';
use strict;
use warnings;
use Setup;
my %ctx = test_init();
require Hydra::Schema;
require Hydra::Model::DB;
use Test2::V0;
my $db = Hydra::Model::DB->new;
hydra_setup($db);
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("broken-constituent", "broken-constituent.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset),               "Evaluating jobs/broken-constituent.nix should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/broken-constituent.nix should not queue any builds");
like(
    $jobset->errormsg,
    qr/^does-not-exist: does not exist$/m,
    "Evaluating jobs/broken-constituent.nix should log an error for does-not-exist");
like(
    $jobset->errormsg,
    qr/^does-not-evaluate: error: assertion 'false' failed$/m,
    "Evaluating jobs/broken-constituent.nix should log an error for does-not-evaluate");
done_testing;