grahamc/hydra - Change D4G756UKRXYU6QEKD6TGN2VAL7PDJMLBMQALHKDPZU3I6U5KVTYQC

hydra-eval-jobs: Identify unexpected errors in handling aggregate jobs

The vague "[json.exception.type_error.302] type must be string, but is null" is absolutely unhelpful in the way Hydra currently handles it on evaluation.

This is handling unexpected errors only; the following commit will handle the specific instance of the previously mentioned error.

Created by Samuel Dionne-Riel on October 25, 2020

D4G756UKRXYU6QEKD6TGN2VAL7PDJMLBMQALHKDPZU3I6U5KVTYQC

Dependencies

In channels

main

upstream

Change contents

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 445 [4.530]

[3.12686]

            // For the error message
            std::string lastTriedJobName = i.key();

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 451 [4.530]

B:BD[3.12790] → [3.12790:13262]

B:BD[3.13262] → [5.0:143]

            if (myArgs.dryRun) {
                for (std::string jobName2 : *named) {
                    auto job2 = state->jobs.find(jobName2);
                    if (job2 == state->jobs.end())
                        throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
                    std::string drvPath2 = (*job2)["drvPath"];
                    job["constituents"].push_back(drvPath2);
                }
            } else {
                auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
                auto drv = store->readDerivation(drvPath);

[3.12790]

[3.13376]

            try {
                if (myArgs.dryRun) {
                    for (std::string jobName2 : *named) {
                        lastTriedJobName = jobName2;
                        auto job2 = state->jobs.find(jobName2);
                        if (job2 == state->jobs.end())
                            throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
                        std::string drvPath2 = (*job2)["drvPath"];
                        job["constituents"].push_back(drvPath2);
                    }
                } else {
                    auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
                    auto drv = store->readDerivation(drvPath);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 465 [4.530]

B:BD[3.13377] → [3.13377:13653]

B:BD[3.13653] → [5.144:463]

∅:D[5.463] → [3.13943:13961]

B:BD[3.13943] → [3.13943:13961]

                for (std::string jobName2 : *named) {
                    auto job2 = state->jobs.find(jobName2);
                    if (job2 == state->jobs.end())
                        throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
                    auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
                    auto drv2 = store->readDerivation(drvPath2);
                    job["constituents"].push_back(store->printStorePath(drvPath2));
                    drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
                }

[3.13377]

[3.13961]

                    for (std::string jobName2 : *named) {
                        lastTriedJobName = jobName2;
                        auto job2 = state->jobs.find(jobName2);
                        if (job2 == state->jobs.end())
                            throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
                        auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
                        auto drv2 = store->readDerivation(drvPath2);
                        job["constituents"].push_back(store->printStorePath(drvPath2));
                        drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
                    }

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 476 [4.530]

B:BD[3.13962] → [5.464:517]

∅:D[5.517] → [3.14038:14166]

B:BD[3.14038] → [3.14038:14166]

B:BD[3.14166] → [6.0:82]

∅:D[6.82] → [3.14232:14370]

B:BD[3.14232] → [3.14232:14370]

B:BD[3.14370] → [5.518:654]

B:BD[5.654] → [2.0:87]

                std::string drvName(drvPath.name());
                assert(hasSuffix(drvName, drvExtension));
                drvName.resize(drvName.size() - drvExtension.size());
                auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));
                auto outPath = store->makeOutputPath("out", h, drvName);
                drv.env["out"] = store->printStorePath(outPath);
                drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
                auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));

[3.13962]

[3.14561]

                    std::string drvName(drvPath.name());
                    assert(hasSuffix(drvName, drvExtension));
                    drvName.resize(drvName.size() - drvExtension.size());
                    auto h = std::get<Hash>(hashDerivationModulo(*store, drv, true));
                    auto outPath = store->makeOutputPath("out", h, drvName);
                    drv.env["out"] = store->printStorePath(outPath);
                    drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } });
                    auto newDrvPath = store->printStorePath(writeDerivation(*store, drv));
                    debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 487 [4.530]

B:BD[3.14562] → [5.655:763]

                debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath);

[3.14562]

[3.14647]

                    job["drvPath"] = newDrvPath;
                    job["outputs"]["out"] = store->printStorePath(outPath);
                }
            } catch (std::exception & e) {
                // Print more information to help debugging.
                printError("Unexpected error in hydra-eval-jobs when handling job '%s', when producing aggregate job '%s':", lastTriedJobName, jobName);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 494 [4.530]

B:BD[3.14648] → [3.14648:14765]

                job["drvPath"] = newDrvPath;
                job["outputs"]["out"] = store->printStorePath(outPath);

[3.14648]

[3.14765]

                // And throw the original exception!
                throw;