grahamc/hydra - Change NJXD2ABJHKH7MBAPEOXQRKC3RK2OVNRO4ND44SOZJECZGCGZK7KQC

Abort unsupported build steps

If we don't see machine that supports a build step for 'max_unsupported_time' seconds, the step is aborted. The default is 0, which is appropriate for Hydra installations that don't provision missing machines dynamically.

(cherry picked from commit f5cdbfe21d930db43d3812c7d8e87746d6378ef9)

Created by Eelco Dolstra on March 26, 2020

NJXD2ABJHKH7MBAPEOXQRKC3RK2OVNRO4ND44SOZJECZGCGZK7KQC

Dependencies

In channels

main

upstream

Change contents

Replacement in src/hydra-queue-runner/builder.cc at line 379 [5.133]

B:BD[5.9302] → [5.9302:9315]

    } else {

[5.9302]

[6.303]

    } else
        failStep(*conn, step, buildId, result, machine, stepFinished, quit);

Replacement in src/hydra-queue-runner/builder.cc at line 382 [5.133]

∅:D[6.304] → [5.9316:9445]

B:BD[5.9316] → [5.9316:9445]

        /* Register failure in the database for all Build objects that
           directly or indirectly depend on this step. */

[6.304]

[5.9445]

    // FIXME: keep stats about aborted steps?
    nrStepsDone++;
    totalStepTime += stepStopTime - stepStartTime;
    totalStepBuildTime += result.stopTime - result.startTime;
    machine->state->nrStepsDone++;
    machine->state->totalStepTime += stepStopTime - stepStartTime;
    machine->state->totalStepBuildTime += result.stopTime - result.startTime;

Replacement in src/hydra-queue-runner/builder.cc at line 390 [5.133]

B:BD[5.9446] → [5.9446:9489]

        std::vector<BuildID> dependentIDs;

[5.9446]

[5.9489]

    if (quit) exit(0); // testing hack; FIXME: this won't run plugins

Replacement in src/hydra-queue-runner/builder.cc at line 392 [5.133]

B:BD[5.9490] → [5.9490:9513]

B:BD[5.9514] → [5.9514:9780]

        while (true) {
            /* Get the builds and steps that depend on this step. */
            std::set<Build::ptr> indirect;
            {
                auto steps_(steps.lock());
                std::set<Step::ptr> steps;
                getDependents(step, indirect, steps);

[5.9490]

[5.9780]

    return sDone;
}

Deletion in src/hydra-queue-runner/builder.cc at line 395 [5.133]

B:BD[5.9781] → [5.9781:10074]

B:BD[5.10074] → [4.5050:5195]

∅:D[4.5195] → [5.10173:10246]

B:BD[5.10173] → [5.10173:10246]

B:BD[5.10273] → [5.10273:10305]

                /* If there are no builds left, delete all referring
                   steps from ‘steps’. As for the success case, we can
                   be certain no new referrers can be added. */
                if (indirect.empty()) {
                    for (auto & s : steps) {
                        printMsg(lvlDebug, "finishing build step ‘%s’",
                            localStore->printStorePath(s->drvPath));
                        steps_->erase(s->drvPath);
                    }
                }
            }

Replacement in src/hydra-queue-runner/builder.cc at line 396 [5.133]

B:BD[6.306] → [6.306:363]

            if (indirect.empty() && stepFinished) break;

[6.306]

[5.10305]

void State::failStep(
    Connection & conn,
    Step::ptr step,
    BuildID buildId,
    const RemoteResult & result,
    Machine::ptr machine,
    bool & stepFinished,
    bool & quit)
{
    /* Register failure in the database for all Build objects that
       directly or indirectly depend on this step. */

Replacement in src/hydra-queue-runner/builder.cc at line 408 [5.133]

B:BD[5.10306] → [5.10306:10359]

B:BD[5.10359] → [7.125:168]

            /* Update the database. */
            {
                auto mc = startDbUpdate();

[5.10306]

[7.168]

    std::vector<BuildID> dependentIDs;

Replacement in src/hydra-queue-runner/builder.cc at line 410 [5.133]

∅:D[7.169] → [5.10359:10398]

B:BD[5.10359] → [5.10359:10398]

                pqxx::work txn(*conn);

[7.169]

[5.10806]

    while (true) {
        /* Get the builds and steps that depend on this step. */
        std::set<Build::ptr> indirect;
        {
            auto steps_(steps.lock());
            std::set<Step::ptr> steps;
            getDependents(step, indirect, steps);

Replacement in src/hydra-queue-runner/builder.cc at line 418 [5.133]

B:BD[5.11220] → [8.2802:3078]

∅:D[8.3078] → [5.11545:11594]

B:BD[5.11545] → [5.11545:11594]

B:BD[5.11594] → [8.3079:3181]

B:BD[8.3181] → [9.2138:2229]

∅:D[9.2229] → [5.11736:11816]

∅:D[8.3266] → [5.11736:11816]

B:BD[5.11736] → [5.11736:11816]

B:BD[5.11816] → [9.2230:2408]

                /* Create failed build steps for every build that
                   depends on this, except when this step is cached
                   and is the top-level of that build (since then it's
                   redundant with the build's isCachedBuild field). */
                for (auto & build2 : indirect) {
                    if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) ||
                        (result.stepStatus != bsCachedFailure && buildId == build2->id) ||
                        build2->finishedInDB)
                        continue;
                    createBuildStep(txn, 0, build2->id, step, machine->sshName,
                        result.stepStatus, result.errorMsg, buildId == build2->id ? 0 : buildId);

[5.10807]

[5.11984]

            /* If there are no builds left, delete all referring
               steps from ‘steps’. As for the success case, we can
               be certain no new referrers can be added. */
            if (indirect.empty()) {
                for (auto & s : steps) {
                    printMsg(lvlDebug, "finishing build step ‘%s’",
                        localStore->printStorePath(s->drvPath));
                    steps_->erase(s->drvPath);

Insertion in src/hydra-queue-runner/builder.cc at line 427 [5.133]
[5.12002]
[5.12002]
```
            }
        }
```

Replacement in src/hydra-queue-runner/builder.cc at line 430 [5.133]

B:BD[5.12212] → [5.12212:12527]

B:BD[5.12527] → [10.0:190]

∅:D[11.184] → [5.12696:12733]

∅:D[10.190] → [5.12696:12733]

B:BD[5.12696] → [5.12696:12733]

B:BD[5.12733] → [8.3520:3660]

∅:D[8.3660] → [5.12855:12940]

B:BD[5.12855] → [5.12855:12940]

B:BD[5.12940] → [8.3661:3740]

∅:D[8.3740] → [5.12996:13050]

B:BD[5.12996] → [5.12996:13050]

                /* Mark all builds that depend on this derivation as failed. */
                for (auto & build2 : indirect) {
                    if (build2->finishedInDB) continue;
                    printMsg(lvlError, format("marking build %1% as failed") % build2->id);
                    txn.parameterized
                        ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0")
                        (build2->id)
                        ((int) (build2->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus()))
                        (result.startTime)
                        (result.stopTime)
                        (result.stepStatus == bsCachedFailure ? 1 : 0).exec();
                    nrBuildsDone++;
                }

[5.12003]

[5.13050]

        if (indirect.empty() && stepFinished) break;

Replacement in src/hydra-queue-runner/builder.cc at line 432 [5.133]

B:BD[5.13051] → [5.13051:13165]

B:BD[5.13165] → [8.3741:3818]

B:BD[8.3818] → [4.5196:5384]

                /* Remember failed paths in the database so that they
                   won't be built again. */
                if (result.stepStatus != bsCachedFailure && result.canCache)
                    for (auto & path : step->drv->outputPaths())
                        txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec();

[5.13051]

[5.13409]

        /* Update the database. */
        {
            auto mc = startDbUpdate();

Replacement in src/hydra-queue-runner/builder.cc at line 436 [5.133]
B:BD[5.13410] → [5.13410:13454]
```
                txn.commit();
            }
```
[5.13410]
[12.352]
```
            pqxx::work txn(conn);
```

Replacement in src/hydra-queue-runner/builder.cc at line 438 [5.133]

B:BD[12.353] → [12.353:386]

            stepFinished = true;

[12.353]

[5.13454]

            /* Create failed build steps for every build that
               depends on this, except when this step is cached
               and is the top-level of that build (since then it's
               redundant with the build's isCachedBuild field). */
            for (auto & build : indirect) {
                if ((result.stepStatus == bsCachedFailure && build->drvPath == step->drvPath) ||
                    ((result.stepStatus != bsCachedFailure && result.stepStatus != bsUnsupported) && buildId == build->id) ||
                    build->finishedInDB)
                    continue;
                createBuildStep(txn,
                    0, build->id, step, machine ? machine->sshName : "",
                    result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId);
            }

Replacement in src/hydra-queue-runner/builder.cc at line 452 [5.133]

B:BD[5.13455] → [5.13455:13841]

            /* Remove the indirect dependencies from ‘builds’. This
               will cause them to be destroyed. */
            for (auto & b : indirect) {
                auto builds_(builds.lock());
                b->finishedInDB = true;
                builds_->erase(b->id);
                dependentIDs.push_back(b->id);
                if (buildOne == b->id) quit = true;

[5.13455]

[5.13841]

            /* Mark all builds that depend on this derivation as failed. */
            for (auto & build : indirect) {
                if (build->finishedInDB) continue;
                printMsg(lvlError, format("marking build %1% as failed") % build->id);
                txn.parameterized
                    ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0")
                    (build->id)
                    ((int) (build->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus()))
                    (result.startTime)
                    (result.stopTime)
                    (result.stepStatus == bsCachedFailure ? 1 : 0).exec();
                nrBuildsDone++;

Deletion in src/hydra-queue-runner/builder.cc at line 465 [5.133]
B:BD[5.13855] → [5.13855:13865]
```
        }
```

Replacement in src/hydra-queue-runner/builder.cc at line 466 [5.133]

B:BD[5.13866] → [5.13866:13945]

B:BD[5.13945] → [3.354:450]

        /* Send notification about this build and its dependents. */
        {
            pqxx::work txn(*conn);
            notifyBuildFinished(txn, buildId, dependentIDs);

[5.13866]

[3.450]

            /* Remember failed paths in the database so that they
               won't be built again. */
            if (result.stepStatus != bsCachedFailure && result.canCache)
                for (auto & path : step->drv->outputPaths())
                    txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec();

Deletion in src/hydra-queue-runner/builder.cc at line 474 [5.133]
B:BD[5.14165] → [5.14165:14171]
```
    }
```

Replacement in src/hydra-queue-runner/builder.cc at line 475 [5.133]

B:BD[5.14172] → [5.14172:14530]

    // FIXME: keep stats about aborted steps?
    nrStepsDone++;
    totalStepTime += stepStopTime - stepStartTime;
    totalStepBuildTime += result.stopTime - result.startTime;
    machine->state->nrStepsDone++;
    machine->state->totalStepTime += stepStopTime - stepStartTime;
    machine->state->totalStepBuildTime += result.stopTime - result.startTime;

[5.14172]

[5.14530]

        stepFinished = true;

Replacement in src/hydra-queue-runner/builder.cc at line 477 [5.133]

B:BD[5.14531] → [13.321:391]

    if (quit) exit(0); // testing hack; FIXME: this won't run plugins

[5.14531]

[5.14570]

        /* Remove the indirect dependencies from ‘builds’. This
           will cause them to be destroyed. */
        for (auto & b : indirect) {
            auto builds_(builds.lock());
            b->finishedInDB = true;
            builds_->erase(b->id);
            dependentIDs.push_back(b->id);
            if (buildOne == b->id) quit = true;
        }
    }

Replacement in src/hydra-queue-runner/builder.cc at line 488 [5.133]

B:BD[5.14571] → [14.297:315]

    return sDone;

[5.14571]

[5.14589]

    /* Send notification about this build and its dependents. */
    {
        pqxx::work txn(conn);
        notifyBuildFinished(txn, buildId, dependentIDs);
        txn.commit();
    }

Insertion in src/hydra-queue-runner/dispatcher.cc at line 302 [5.14628]
[15.4651]
[5.19812]
```
    abortUnsupported();
```

Insertion in src/hydra-queue-runner/dispatcher.cc at line 317 [5.14628]

[5.19999]

[16.1304]


void State::abortUnsupported()
{
    /* Make a copy of 'runnable' and 'machines' so we don't block them
       very long. */
    auto runnable2 = *runnable.lock();
    auto machines2 = *machines.lock();
    system_time now = std::chrono::system_clock::now();
    auto now2 = time(0);

Insertion in src/hydra-queue-runner/dispatcher.cc at line 329 [5.14628]
[16.1305]
[16.1305]
```
    std::unordered_set<Step::ptr> aborted;
```

Insertion in src/hydra-queue-runner/dispatcher.cc at line 331 [5.14628]

[16.1306]

    for (auto & wstep : runnable2) {
        auto step(wstep.lock());
        if (!step) continue;
        bool supported = false;
        for (auto & machine : machines2) {
            if (machine.second->supportsStep(step)) {
                step->state.lock()->lastSupported = now;
                supported = true;
                break;
            }
        }
        if (!supported
            && std::chrono::duration_cast<std::chrono::seconds>(now - step->state.lock()->lastSupported).count() >= maxUnsupportedTime)
        {
            printError("aborting unsupported build step '%s' (type '%s')",
                localStore->printStorePath(step->drvPath),
                step->systemType);
            aborted.insert(step);
            auto conn(dbPool.get());
            std::set<Build::ptr> dependents;
            std::set<Step::ptr> steps;
            getDependents(step, dependents, steps);
            /* Maybe the step got cancelled. */
            if (dependents.empty()) continue;
            /* Find the build that has this step as the top-level (if
               any). */
            Build::ptr build;
            for (auto build2 : dependents) {
                if (build2->drvPath == step->drvPath)
                    build = build2;
            }
            if (!build) build = *dependents.begin();
            bool stepFinished = false;
            bool quit = false;
            failStep(
                *conn, step, build->id,
                RemoteResult {
                    .stepStatus = bsUnsupported,
                    .errorMsg = fmt("unsupported system type '%s'",
                        step->systemType),
                    .startTime = now2,
                    .stopTime = now2,
                },
                nullptr, stepFinished, quit);
            if (quit) exit(1);
        }
    }
    /* Clean up 'runnable'. */
    {
        auto runnable_(runnable.lock());
        for (auto i = runnable_->begin(); i != runnable_->end(); ) {
            if (aborted.count(i->lock()))
                i = runnable_->erase(i);
            else
                ++i;
        }
    }
}

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 49 [18.4840]
[2.43]
[17.0]
```
    , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0))
```
Replacement in src/hydra-queue-runner/state.hh at line 71 [20.1128]
B:BD[19.3662] → [8.4211:4241]
```
    BuildStatus buildStatus()
```
[19.3662]
[19.3682]
```
    BuildStatus buildStatus() const
```

Insertion in src/hydra-queue-runner/state.hh at line 201 [20.1128]

[21.1366]

[20.3082]


        /* The time that we last saw a machine that supports this
           step. */
        system_time lastSupported = std::chrono::system_clock::now();

Insertion in src/hydra-queue-runner/state.hh at line 310 [20.1128]

[20.4357]

[22.404]

    /* Time in seconds before unsupported build steps are aborted. */
    const unsigned int maxUnsupportedTime = 0;

Insertion in src/hydra-queue-runner/state.hh at line 492 [20.1128]

[20.8032]


    void failStep(
        Connection & conn,
        Step::ptr step,
        BuildID buildId,
        const RemoteResult & result,
        Machine::ptr machine,
        bool & stepFinished,
        bool & quit);

Insertion in src/hydra-queue-runner/state.hh at line 516 [20.1128]
[20.8187]
[23.3028]
```
    void abortUnsupported();
```