grahamc/hydra - Change LVQXQIYA7QMLVYOANYEFHDBTFAOSE3D2IYAVOG2DXURTASRCUNYQC

Kill active build steps when builds are cancelled

We now kill active build steps when there are no more referring builds. This is useful e.g. for preventing cancelled multi-hour TPC-H benchmark runs from hogging build machines.

Created by Eelco Dolstra on October 31, 2016

LVQXQIYA7QMLVYOANYEFHDBTFAOSE3D2IYAVOG2DXURTASRCUNYQC

Dependencies

In channels

upstream

main

Change contents

Insertion in src/hydra-queue-runner/builder.cc at line 15 [13.133]
[5.23]
[13.350]
```
    reservation->threadId = pthread_self();
```

Replacement in src/hydra-queue-runner/builder.cc at line 18 [13.133]

B:BD[13.351] → [13.351:388]

    MaintainCount mc(nrActiveSteps);

[13.351]

[14.58]

    activeSteps_.lock()->insert(reservation);
    Finally removeActiveStep([&]() {
        reservation->threadId = -1;
        activeSteps_.lock()->erase(reservation);
    });

Replacement in src/hydra-queue-runner/builder.cc at line 73 [13.133]

B:BD[13.2097] → [13.2097:2164]

       thousands of builds), so we don't. */
    Build::ptr build;

[13.2097]

[13.2164]

       thousands of builds), so we don't.
       We don't keep a Build::ptr here to allow
       State::processQueueChange() to detect whether a step can be
       cancelled (namely if there are no more Builds referring to
       it). */
    BuildID buildId;
    Path buildDrvPath;
    unsigned int maxSilentTime, buildTimeout;

Insertion in src/hydra-queue-runner/builder.cc at line 102 [13.133]
[13.3048]
[15.0]
```
        Build::ptr build;
```

Insertion in src/hydra-queue-runner/builder.cc at line 112 [13.133]

[13.3215]

        buildId = build->id;
        buildDrvPath = build->drvPath;
        maxSilentTime = build->maxSilentTime;
        buildTimeout = build->buildTimeout;

Replacement in src/hydra-queue-runner/builder.cc at line 118 [13.133]

B:BD[13.3327] → [13.3327:3414]

            % step->drvPath % machine->sshName % build->id % (dependents.size() - 1));

[13.3327]

[13.3414]

            % step->drvPath % machine->sshName % buildId % (dependents.size() - 1));

Replacement in src/hydra-queue-runner/builder.cc at line 121 [13.133]

B:BD[13.3421] → [4.0:74]

    bool quit = build->id == buildOne && step->drvPath == build->drvPath;

[13.3421]

[13.3460]

    bool quit = buildId == buildOne && step->drvPath == buildDrvPath;

Replacement in src/hydra-queue-runner/builder.cc at line 132 [13.133]

B:BD[16.124] → [12.0:86]

            printError("marking step %d of build %d as orphaned", stepNr, build->id);

[16.124]

            printError("marking step %d of build %d as orphaned", stepNr, buildId);

Replacement in src/hydra-queue-runner/builder.cc at line 134 [13.133]

B:BD[16.179] → [16.179:235]

            orphanedSteps_->emplace(build->id, stepNr);

[16.179]

[16.235]

            orphanedSteps_->emplace(buildId, stepNr);

Replacement in src/hydra-queue-runner/builder.cc at line 151 [13.133]

B:BD[13.4030] → [17.0:100]

            stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bsBusy);

[13.4030]

[13.4224]

            stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);

Replacement in src/hydra-queue-runner/builder.cc at line 158 [13.133]

B:BD[13.4376] → [18.1330:1432]

            buildRemote(destStore, machine, step, build->maxSilentTime, build->buildTimeout, result);

[13.4376]

[19.2415]

            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, result);

Insertion in src/hydra-queue-runner/builder.cc at line 165 [13.133]

[7.36]

[13.4597]

        } catch (__cxxabiv1::__forced_unwind & e) {
            /* The queue monitor thread cancelled this step. */
            try {
                printInfo("marking step %d of build %d as succeeded", stepNr, buildId);
                pqxx::work txn(*conn);
                finishBuildStep(txn, result.startTime, time(0), result.overhead, buildId,
                    stepNr, machine->sshName, bsCancelled, "");
                txn.commit();
                stepFinished = true;
            } catch (...) {
                ignoreException();
            }
            throw;

Replacement in src/hydra-queue-runner/builder.cc at line 204 [13.133]

B:BD[11.149] → [11.149:225]

            logCompressorQueue_->push({build->id, stepNr, result.logFile});

[11.149]

[13.4984]

            logCompressorQueue_->push({buildId, stepNr, result.logFile});

Replacement in src/hydra-queue-runner/builder.cc at line 224 [13.133]

B:BD[13.5536] → [20.434:530]

            finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, build->id,

[13.5536]

[8.0]

            finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, buildId,

Replacement in src/hydra-queue-runner/builder.cc at line 280 [13.133]

B:BD[20.620] → [17.173:242]

                    build->id, stepNr, machine->sshName, bsSuccess);

[20.620]

[13.7478]

                    buildId, stepNr, machine->sshName, bsSuccess);

Replacement in src/hydra-queue-runner/builder.cc at line 284 [13.133]

∅:D[9.131] → [19.2665:2748]

B:BD[13.7519] → [19.2665:2748]

                    markSucceededBuild(txn, b, res, build != b || result.isCached,

[9.131]

[13.7622]

                    markSucceededBuild(txn, b, res, buildId != b->id || result.isCached,

Replacement in src/hydra-queue-runner/builder.cc at line 377 [13.133]

B:BD[19.3181] → [19.3181:3266]

                        (result.stepStatus != bsCachedFailure && build == build2) ||

[19.3181]

[13.11736]

                        (result.stepStatus != bsCachedFailure && buildId == build2->id) ||

Replacement in src/hydra-queue-runner/builder.cc at line 380 [13.133]

B:BD[13.11816] → [13.11816:11892]

B:BD[13.11892] → [19.3267:3361]

                    createBuildStep(txn, 0, build2, step, machine->sshName,
                        result.stepStatus, result.errorMsg, build == build2 ? 0 : build->id);

[13.11816]

[13.11984]

                    createBuildStep(txn, 0, build2->id, step, machine->sshName,
                        result.stepStatus, result.errorMsg, buildId == build2->id ? 0 : buildId);

Replacement in src/hydra-queue-runner/builder.cc at line 387 [13.133]

B:BD[20.784] → [19.3421:3519]

                        build->id, stepNr, machine->sshName, result.stepStatus, result.errorMsg);

[20.784]

[12.478]

                        buildId, stepNr, machine->sshName, result.stepStatus, result.errorMsg);

Replacement in src/hydra-queue-runner/builder.cc at line 429 [13.133]

B:BD[13.14020] → [11.342:468]

            notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::BuildFinished, build->id, dependentIDs});

[13.14020]

[13.14107]

            notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::BuildFinished, buildId, dependentIDs});

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 227 [21.4840]

B:BD[21.8075] → [11.469:540]

unsigned int State::allocBuildStep(pqxx::work & txn, Build::ptr build)

[21.8075]

[21.8254]

unsigned int State::allocBuildStep(pqxx::work & txn, BuildID buildId)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 233 [21.4840]

∅:D[22.198] → [21.8256:8363]

B:BD[21.8256] → [21.8256:8363]

    auto res = txn.parameterized("select max(stepnr) from BuildSteps where build = $1")(build->id).exec();

[22.198]

[23.63]

    auto res = txn.parameterized("select max(stepnr) from BuildSteps where build = $1")(buildId).exec();

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 238 [21.4840]

B:BD[23.129] → [11.541:647]

unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,

[23.129]

[17.452]

unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step,

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 241 [21.4840]
B:BD[23.339] → [11.648:702]
```
    unsigned int stepNr = allocBuildStep(txn, build);
```
[23.339]
[21.8431]
```
    auto stepNr = allocBuildStep(txn, buildId);
```
Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 245 [21.4840]
B:BD[24.578] → [23.385:405]
```
        (build->id)
```
[24.578]
[23.405]
```
        (buildId)
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 261 [21.4840]

B:BD[21.9164] → [21.9164:9238]

            (build->id)(stepNr)(output.first)(output.second.path).exec();

[21.9164]

[21.9238]

            (buildId)(stepNr)(output.first)(output.second.path).exec();

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 287 [21.4840]
B:BD[23.691] → [23.691:736]
```
    int stepNr = allocBuildStep(txn, build);
```
[23.691]
[23.736]
```
    auto stepNr = allocBuildStep(txn, build->id);
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 577 [21.4840]

B:BD[25.1115] → [25.1115:1166]

        root.attr("nrActiveSteps", nrActiveSteps);

[25.1115]

[25.1166]

        root.attr("nrActiveSteps", activeSteps_.lock()->size());

Insertion in src/hydra-queue-runner/queue-monitor.cc at line 4 [13.20067]
[23.1279]
[23.1280]
```
#include <cstring>
```

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 185 [13.20067]

B:BD[10.1858] → [10.1858:1956]

                createBuildStep(txn, 0, build, ex.step, "", bsCachedFailure, "", propagatedFrom);

[10.1858]

[10.1956]

                createBuildStep(txn, 0, build->id, ex.step, "", bsCachedFailure, "", propagatedFrom);

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 317 [13.20067]
B:BD[13.29011] → [13.29011:29044]
```
    auto builds_(builds.lock());
```
[13.29011]
[13.29044]
```
    {
        auto builds_(builds.lock());
```

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 320 [13.20067]

B:BD[13.29045] → [13.29045:29106]

B:BD[13.29106] → [26.1591:1672]

∅:D[26.1672] → [13.29167:29360]

B:BD[13.29167] → [13.29167:29360]

B:BD[13.29360] → [26.1673:1695]

    for (auto i = builds_->begin(); i != builds_->end(); ) {
        auto b = currentIds.find(i->first);
        if (b == currentIds.end()) {
            printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first);
            i = builds_->erase(i);
            // FIXME: ideally we would interrupt active build steps here.
            continue;

[13.29045]

[26.1695]

        for (auto i = builds_->begin(); i != builds_->end(); ) {
            auto b = currentIds.find(i->first);
            if (b == currentIds.end()) {
                printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first);
                i = builds_->erase(i);
                // FIXME: ideally we would interrupt active build steps here.
                continue;
            }
            if (i->second->globalPriority < b->second) {
                printMsg(lvlInfo, format("priority of build %1% increased") % i->first);
                i->second->globalPriority = b->second;
                i->second->propagatePriorities();
            }
            ++i;

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 335 [13.20067]

B:BD[26.1705] → [26.1705:1940]

        if (i->second->globalPriority < b->second) {
            printMsg(lvlInfo, format("priority of build %1% increased") % i->first);
            i->second->globalPriority = b->second;
            i->second->propagatePriorities();

[26.1705]

[26.1940]

    }
    {
        auto activeSteps(activeSteps_.lock());
        for (auto & activeStep : *activeSteps) {
            auto threadId = activeStep->threadId; // FIXME: use Sync or atomic?
            if (threadId == 0) continue;
            std::set<Build::ptr> dependents;
            std::set<Step::ptr> steps;
            getDependents(activeStep->step, dependents, steps);
            if (!dependents.empty()) continue;
            printInfo("cancelling thread for build step ‘%s’", activeStep->step->drvPath);
            int err = pthread_cancel(threadId);
            if (err)
                printError("error cancelling thread for build step ‘%s’: %s",
                    activeStep->step->drvPath, strerror(err));

Deletion in src/hydra-queue-runner/queue-monitor.cc at line 355 [13.20067]
B:BD[26.1950] → [26.1950:1963]
```
        ++i;
```
Insertion in src/hydra-queue-runner/state.hh at line 31 [27.1128]
[27.1660]
[17.912]
```
    bsCancelled = 4,
```
Deletion in src/hydra-queue-runner/state.hh at line 300 [27.1128]
B:BD[27.5630] → [27.5630:5660]
```
    counter nrActiveSteps{0};
```
Insertion in src/hydra-queue-runner/state.hh at line 373 [27.1128]
[14.2908]
[14.2908]
```
        pthread_t threadId = 0;
        bool cancelled = false;
```
Insertion in src/hydra-queue-runner/state.hh at line 378 [27.1128]
[14.3027]
[2.1480]
```
    nix::Sync<std::set<std::shared_ptr<MachineReservation>>> activeSteps_;
```

Replacement in src/hydra-queue-runner/state.hh at line 420 [27.1128]

B:BD[23.3075] → [11.3251:3320]

    unsigned int allocBuildStep(pqxx::work & txn, Build::ptr build);

[23.3075]

[27.7026]

    unsigned int allocBuildStep(pqxx::work & txn, BuildID buildId);

Replacement in src/hydra-queue-runner/state.hh at line 422 [27.1128]

B:BD[27.7027] → [11.3321:3424]

    unsigned int createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,

[27.7027]

[17.1029]

    unsigned int createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step,

Replacement in src/root/build.tt at line 26 [30.7350]

∅:D[28.53] → [29.11819:11941]

∅:D[3.85] → [29.11819:11941]

∅:D[31.2133] → [29.11819:11941]

B:BD[29.11819] → [29.11819:11941]

        [% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %]

[3.85]

[28.54]

        [% IF ( type == "All" ) || ( type == "Failed" && step.busy == 0 && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %]

Replacement in src/root/build.tt at line 52 [30.7350]

B:BD[32.932] → [6.0:240]

            <td>[% IF step.busy == 1 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>

[32.932]

[32.985]

            <td>[% IF step.busy == 1 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>

Insertion in src/root/build.tt at line 60 [30.7350]

[33.6911]

[34.0]

              [% ELSIF step.status == 4 %]
                <span class="error">Cancelled</span>

Replacement in src/root/build.tt at line 247 [30.7350]

B:BD[35.70] → [31.2322:2395]

      [% IF steps && build.buildstatus != 0 && build.buildstatus != 6 %]

[35.70]

[32.5185]

      [% IF steps && build.buildstatus != 0 && build.buildstatus != 4 && build.buildstatus != 6 %]

Replacement in src/sql/hydra.sql at line 206 [36.1]

B:BD[17.1560] → [17.1560:1637]

    --   4 = build cancelled (removed from queue; never built) [builds only]

[17.1560]

[17.1637]

    --   4 = build or step cancelled