Fix race between the queue monitor and the builder threads

Dependencies

• [2] HLSHCK3C Support requiredSystemFeatures
• [3] NNOCZ4RO hydra-queue-runner: Improve dispatcher
• [4] UPN2SAML Acquire exclusive table lock earlier
• [5] TM6WKSP3 hydra-queue-runner: Set isCachedBuild
• [6] QJRDO2B4 Simplify retry handling
• [7] RYTQLATY Keep track of failed paths in the Hydra database
• [8] 22LDPAIP Check non-runnable steps for unsupported system type
• [9] NJJ7H64S Very basic multi-threaded queue runner
• [10] YZAI5GQU Implement a database connection pool
• [11] 2IQRXLWE Support cancelling builds
• [12] HHOMBU7G hydra-queue-runner: Implement timeouts
• [13] 5AIYUMTB Basic remote building
• [14] KBZHIGLG Record the machine used for a build step
• [15] 62MQPRXC Pass null values to libpqxx properly
• [16] OCZ4LSGG Automatically retry aborted builds
• [17] PQFOMNTL hydra-queue-runner: More stats
• [18] IWB3F4Z6 Fail builds with previously failed steps early
• [19] JAUB2FT5 getQueuedBuilds(): Handle dependent builds first
• [20] LJILHOJ7 Create BuildSteps race-free
• [21] ENXUSMSV Make concurrency more robust
• [22] GKZN4UV7 Make the queue monitor more robust, and better debug output
• [23] 24BMQDZA Start of single-process hydra-queue-runner
• [*] N5O7VEEO Immediately abort builds that require an unsupported system type
• [*] UQQ4IL55 Add a error type for "unsupported system type"

Change contents

• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 122
  [7.208]→[7.5623:5647](∅→∅),[7.5623]→[7.5623:5647](∅→∅)
  −

  − bool finishedInDB;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 123
  [7.5648]→[7.5648:5686](∅→∅)
  − Build() : finishedInDB(false) { }
  [7.5648]

  [7.209]
  + std::atomic_bool finishedInDB{false};
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 159
  [7.5906]→[7.634:657](∅→∅)
  − Sync<State> state;
  [7.5906]

  [7.5852]
  + std::atomic_bool created{false}; // debugging

  + std::atomic_bool finished{false}; // debugging
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 162
  [7.5853]→[7.5853:5885](∅→∅)
  − std::atomic_bool destroyed;
  [7.5853]

  [7.5885]
  + Sync<State> state;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 164
  [7.5886]→[7.5886:5920](∅→∅),[7.657]→[7.5993:5994](∅→∅),[7.5920]→[7.5993:5994](∅→∅),[7.5993]→[7.5993:5994](∅→∅),[7.5994]→[7.1434:1450](∅→∅)
  − Step() : destroyed(false) { }

  −

  − ~Step() { }
  [7.5886]

  [7.5927]
  + ~Step()

  + {

  + printMsg(lvlError, format("destroying step %1%") % drvPath);

  + }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 283
  [7.1249]

  [7.0]
  + Build::ptr referringBuild, Step::ptr referringStep,
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 286
  [7.7092]→[7.7092:7263](∅→∅)
  − void destroyStep(Step::ptr step, bool proceed);

  −

  − /* Get the builds that depend on the given step. */

  − std::set<Build::ptr> getDependentBuilds(Step::ptr step);

  −
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 524
  [7.430]

  [7.10645]
  + assert(!build->finishedInDB);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 538
  [7.354]→[7.128:211](∅→∅),[7.128]→[7.128:211](∅→∅)
  − Step::ptr step = createStep(store, build->drvPath, newSteps, newRunnable);
  [7.354]

  [7.355]
  + Step::ptr step = createStep(store, build->drvPath, build, 0, newSteps, newRunnable);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 560
  [7.9878]→[7.971:1064](∅→∅),[7.1064]→[7.11213:11214](∅→∅),[7.2064]→[7.11213:11214](∅→∅),[7.9949]→[7.11213:11214](∅→∅),[7.11213]→[7.11213:11214](∅→∅)
  − printMsg(lvlInfo, format("marking build %1% as cached successful") % build->id);

  −
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 565
  [25.273]

  [7.924]
  + build->finishedInDB = true;

  +
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 603
  [4.72]

  [26.310]
  + assert(!build->finishedInDB);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 625
  [7.3380]→[7.3380:3424](∅→∅)
  − auto step_(step->state.lock());
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 626
  [7.3467]→[7.3467:3511](∅→∅)
  − step_->builds.push_back(build);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 631
  [7.442]→[7.10110:10111](∅→∅),[7.1125]→[7.10110:10111](∅→∅),[7.10110]→[7.10110:10111](∅→∅),[7.10111]→[7.3558:3806](∅→∅),[7.3558]→[7.3558:3806](∅→∅)
  −

  − /* Prior to this, the build is not visible to

  − getDependentBuilds(). Now it is, so the build can be

  − failed if a dependency fails. (It can't succeed right away

  − because its top-level is not runnable yet). */

  −
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 680
  [7.4065]

  [7.443]
  + Build::ptr referringBuild, Step::ptr referringStep,
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 683
  [7.11566]→[7.4105:4159](∅→∅)
  − /* Check if the requested step already exists. */
  [7.11566]

  [7.4159]
  + /* Check if the requested step already exists. If not, create a

  + new step. In any case, make the step reachable from

  + referringBuild or referringStep. This is done atomically (with

  + ‘steps’ locked), to ensure that this step can never become

  + reachable from a new build after doBuildStep has removed it

  + from ‘steps’. */

  + Step::ptr step;

  + bool isNew = false;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 693
  [7.4200]

  [7.4200]
  +

  + /* See if the step already exists in ‘steps’ and is not

  + stale. */
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 698
  [7.4280]→[7.4280:4325](∅→∅)
  − auto step = prev->second.lock();
  [7.4280]

  [7.4325]
  + step = prev->second.lock();
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 701
  [7.4450]→[7.4450:4543](∅→∅)
  − if (step) return step;

  − steps_->erase(drvPath); // remove stale entry
  [7.4450]

  [7.4543]
  + if (!step) steps_->erase(drvPath); // remove stale entry

  + }

  +

  + /* If it doesn't exist, create it. */

  + if (!step) {

  + step = std::make_shared<Step>();

  + step->drvPath = drvPath;

  + isNew = true;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 710
  [7.4553]

  [7.4553]
  +

  + auto step_(step->state.lock());

  +

  + if (referringBuild)

  + step_->builds.push_back(referringBuild);

  +

  + if (referringStep)

  + step_->rdeps.push_back(referringStep);

  +

  + (*steps_)[drvPath] = step;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 724
  [7.11732]→[7.4560:4602](∅→∅),[7.4602]→[7.11762:11791](∅→∅),[7.11762]→[7.11762:11791](∅→∅)
  − auto step = std::make_shared<Step>();

  − step->drvPath = drvPath;
  [7.11732]

  [7.11791]
  + if (!isNew) {

  + assert(step->created);

  + return step;

  + }

  +

  + /* Initialize the step. Note that the step may be visible in

  + ‘steps’ before this point, but that doesn't matter because

  + it's not runnable yet, and other threads won't make it

  + runnable while step->created == false. */
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 739
  [2.714]→[7.515:542](∅→∅),[7.11832]→[7.515:542](∅→∅)
  − newSteps.insert(step);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 754
  [7.1430]

  [7.12240]
  + newSteps.insert(step);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 757
  [7.12286]→[7.4603:4629](∅→∅)
  − bool hasDeps = false;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 758
  [7.12329]→[7.543:618](∅→∅)
  − Step::ptr dep = createStep(store, i.first, newSteps, newRunnable);
  [7.12329]

  [7.12374]
  + auto dep = createStep(store, i.first, 0, step, newSteps, newRunnable);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 760
  [7.12393]→[7.4696:4724](∅→∅)
  − hasDeps = true;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 761
  [7.4768]→[7.4768:4810](∅→∅)
  − auto dep_(dep->state.lock());
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 762
  [7.4847]→[7.4847:4888](∅→∅)
  − dep_->rdeps.push_back(step);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 763
  [7.12479]→[7.12479:12486](∅→∅),[7.12486]→[7.4889:5027](∅→∅),[7.5027]→[7.12513:12514](∅→∅),[7.12513]→[7.12513:12514](∅→∅),[7.12514]→[7.5028:5072](∅→∅),[7.2250]→[7.12565:12643](∅→∅),[7.5072]→[7.12565:12643](∅→∅),[7.12565]→[7.12565:12643](∅→∅),[7.12643]→[7.10112:10174](∅→∅),[7.10174]→[7.1431:1514](∅→∅),[7.1514]→[7.5155:5156](∅→∅),[7.5155]→[7.5155:5156](∅→∅),[7.5156]→[7.352:372](∅→∅),[7.372]→[7.5156:5235](∅→∅),[7.5156]→[7.5156:5235](∅→∅)
  − }

  −

  − {

  − auto steps_(steps.lock());

  − assert(steps_->find(drvPath) == steps_->end());

  − (*steps_)[drvPath] = step;

  − }

  −

  − if (!hasDeps) newRunnable.insert(step);

  −

  − return step;

  − }

  −

  −

  − void State::destroyStep(Step::ptr step, bool proceed)

  − {

  − if (step->destroyed) return;

  − step->destroyed = true;

  −

  − printMsg(lvlDebug, format("destroying build step ‘%1%’") % step->drvPath);

  −

  − nrStepsDone++;

  −

  − {

  − auto steps_(steps.lock());

  − steps_->erase(step->drvPath);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 765
  [7.5242]→[7.5242:5278](∅→∅)
  − std::vector<Step::wptr> rdeps;

  −
  [7.5242]

  [7.5278]
  + /* If the step has no (remaining) dependencies, make it

  + runnable. */
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 769
  [7.5324]→[7.5324:5354](∅→∅),[7.5354]→[7.12675:12676](∅→∅),[7.12675]→[7.12675:12676](∅→∅),[7.12676]→[7.5355:5608](∅→∅)
  − rdeps = step_->rdeps;

  −

  − /* Sanity checks. */

  − for (auto & build_ : step_->builds) {

  − auto build = build_.lock();

  − if (!build) continue;

  − assert(build->drvPath == step->drvPath);

  − assert(build->finishedInDB);

  − }
  [7.5324]

  [7.5608]
  + assert(!step->created);

  + step->created = true;

  + if (step_->deps.empty())

  + newRunnable.insert(step);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 775
  [7.5615]→[7.5615:5648](∅→∅),[7.5648]→[7.12715:12778](∅→∅),[7.12715]→[7.12715:12778](∅→∅),[7.12778]→[7.5649:5879](∅→∅),[7.5879]→[7.12869:12991](∅→∅),[7.12869]→[7.12869:12991](∅→∅),[7.12991]→[7.5880:5906](∅→∅),[7.5906]→[7.2251:2287](∅→∅),[7.13027]→[7.2251:2287](∅→∅),[7.2287]→[7.13066:13081](∅→∅),[7.13066]→[7.13066:13081](∅→∅),[7.13081]→[7.5907:6020](∅→∅),[7.6020]→[7.13177:13221](∅→∅),[7.13177]→[7.13177:13221](∅→∅)
  − for (auto & rdep_ : rdeps) {

  − auto rdep = rdep_.lock();

  − if (!rdep) continue;

  − bool runnable = false;

  − {

  − auto rdep_(rdep->state.lock());

  − assert(has(rdep_->deps, step));

  − rdep_->deps.erase(step);

  − if (rdep_->deps.empty()) runnable = true;

  − }

  − if (proceed) {

  − /* If this rdep has no other dependencies, then we can now

  − build it. */

  − if (runnable)

  − makeRunnable(rdep);

  − } else

  − /* If ‘step’ failed or was cancelled, then delete all

  − dependent steps as well. */

  − destroyStep(rdep, false);

  − }
  [7.5615]

  [7.13421]
  + return step;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 779
  [7.13425]→[7.13425:13488](∅→∅)
  − std::set<Build::ptr> State::getDependentBuilds(Step::ptr step)
  [7.13425]

  [7.13488]
  + /* Get the steps and unfinished builds that depend on the given step. */

  + void getDependents(Step::ptr step, std::set<Build::ptr> & builds, std::set<Step::ptr> & steps)
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 782
  [7.13490]→[7.13490:13551](∅→∅)
  − std::set<Step::ptr> done;

  − std::set<Build::ptr> res;

  −
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 785
  [7.13628]→[7.6021:6058](∅→∅),[7.6058]→[7.13679:13706](∅→∅),[7.13679]→[7.13679:13706](∅→∅)
  − if (has(done, step)) return;

  − done.insert(step);
  [7.13628]

  [7.6059]
  + if (has(steps, step)) return;

  + steps.insert(step);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 795
  [7.6248]→[7.6248:6296](∅→∅)
  − if (build_) res.insert(build_);
  [7.6248]

  [7.6296]
  + if (build_ && !build_->finishedInDB) builds.insert(build_);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 810
  [7.13998]→[7.13998:14015](∅→∅)
  −

  − return res;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 819
  [7.6604]

  [7.6604]
  + assert(step->created);

  + assert(!step->finished);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 884
  [3.2004]→[3.2004:2091](∅→∅)
  − printMsg(lvlDebug, format("%1% runnable builds") % runnable_->size());
  [3.2004]

  [3.2091]
  + //printMsg(lvlDebug, format("%1% runnable builds") % runnable_->size());
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 995
  [7.14308]

  [7.14341]
  + {

  + auto step_(step->state.lock());

  + assert(step->created);

  + assert(!step->finished);

  + }

  +
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1011
  [7.7234]→[7.7234:7286](∅→∅)
  − auto dependents = getDependentBuilds(step);
  [7.7234]

  [7.14856]
  + std::set<Build::ptr> dependents;

  + std::set<Step::ptr> steps;

  + getDependents(step, dependents, steps);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1096
  [7.1530]→[7.8142:8563](∅→∅),[7.4555]→[7.8142:8563](∅→∅),[7.16048]→[7.8142:8563](∅→∅),[7.8563]→[6.510:575](∅→∅)
  − /* Remove this step. After this, incoming builds that depend on

  − drvPath will either see that the output paths exist, or will

  − create a new build step for drvPath. The latter is fine - it

  − won't conflict with this one, because we're removing it. In any

  − case, the set of dependent builds for ‘step’ can't increase

  − anymore because ‘step’ is no longer visible to createStep(). */

  − auto steps_(steps.lock());

  − steps_->erase(step->drvPath);
  [7.4555]

  [7.8648]
  + if (result.status == RemoteResult::rrSuccess) {

  +

  + /* Register success in the database for all Build objects that

  + have this step as the top-level step. Since the queue

  + monitor thread may be creating new referring Builds

  + concurrently, and updating the database may fail, we do

  + this in a loop, marking all known builds, repeating until

  + there are no unmarked builds.

  + */

  + while (true) {

  +

  + /* Get the builds that have this one as the top-level. */

  + std::vector<Build::ptr> direct;

  + {

  + auto steps_(steps.lock());

  + auto step_(step->state.lock());

  +

  + for (auto & b_ : step_->builds) {

  + auto b = b_.lock();

  + if (b && !b->finishedInDB) direct.push_back(b);

  + }

  +

  + /* If there are no builds left to update in the DB,

  + then we're done. Delete the step from

  + ‘steps’. Since we've been holding the ‘steps’ lock,

  + no new referrers can have been added in the

  + meantime or be added afterwards. */

  + if (direct.empty()) {

  + printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath);

  + nrStepsDone++;

  + steps_->erase(step->drvPath);

  + break;

  + }

  + }

  +

  + /* Update the database. */

  + {

  + pqxx::work txn(*conn);

  +

  + finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, machine->sshName, bssSuccess);

  +

  + for (auto & b : direct)

  + markSucceededBuild(txn, b, res, build != b,

  + result.startTime, result.stopTime);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1141
  [7.8649]→[7.8649:8746](∅→∅)
  − /* Get the final set of dependent builds. */

  − auto dependents = getDependentBuilds(step);
  [7.8649]

  [7.16115]
  + txn.commit();

  + }
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1144
  [7.16116]→[7.8747:8780](∅→∅),[7.8780]→[7.16116:16122](∅→∅),[7.16116]→[7.16116:16122](∅→∅),[7.16122]→[7.8781:8953](∅→∅)
  − std::set<Build::ptr> direct;

  − {

  − auto step_(step->state.lock());

  − for (auto & build : step_->builds) {

  − auto build_ = build.lock();

  − if (build_) direct.insert(build_);
  [7.16116]

  [7.8953]
  + /* Remove the direct dependencies from ‘builds’. This will

  + cause them to be destroyed. */

  + for (auto & b : direct) {

  + auto builds_(builds.lock());

  + b->finishedInDB = true;

  + builds_->erase(b->id);

  + }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1152
  [7.8963]→[7.8963:8969](∅→∅)
  − }
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1153
  [7.8970]→[7.8970:9007](∅→∅),[7.9007]→[7.740:771](∅→∅)
  − /* Update the database. */

  − {

  − pqxx::work txn(*conn);
  [7.8970]

  [7.16152]
  + /* Wake up any dependent steps that have no other

  + dependencies. */

  + {

  + auto step_(step->state.lock());

  + for (auto & rdepWeak : step_->rdeps) {

  + auto rdep = rdepWeak.lock();

  + if (!rdep) continue;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1161
  [7.16153]→[7.13798:13854](∅→∅)
  − if (result.status == RemoteResult::rrSuccess) {
  [7.16153]

  [7.16176]
  + bool runnable = false;

  + {

  + auto rdep_(rdep->state.lock());

  + rdep_->deps.erase(step);

  + if (rdep_->deps.empty()) runnable = true;

  + }
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1168
  [7.16177]→[7.1113:1231](∅→∅)
  − finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, machine->sshName, bssSuccess);
  [7.16177]

  [7.16252]
  + if (runnable) makeRunnable(rdep);

  + }

  + }
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1172
  [7.16253]→[7.16253:16358](∅→∅),[7.16358]→[7.9008:9047](∅→∅),[7.9047]→[5.0:124](∅→∅)
  − /* Mark all builds of which this derivation is the top

  − level as succeeded. */

  − for (auto build2 : direct)

  − markSucceededBuild(txn, build2, res, build != build2,

  − result.startTime, result.stopTime);
  [7.16253]

  [7.16587]
  + } else {
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1174
  [7.16588]→[7.16588:16605](∅→∅),[7.16605]→[7.1531:1563](∅→∅)
  − } else {

  − /* Failure case. */
  [7.16588]

  [7.199]
  + /* Register failure in the database for all Build objects that

  + directly or indirectly depend on this step. */
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1177
  [7.200]→[7.4575:4613](∅→∅),[7.4613]→[7.1238:1417](∅→∅),[7.1417]→[7.4703:4749](∅→∅),[7.4703]→[7.4703:4749](∅→∅),[7.4749]→[7.1418:1600](∅→∅)
  − BuildStatus buildStatus =

  − result.status == RemoteResult::rrPermanentFailure ? bsFailed :

  − result.status == RemoteResult::rrTimedOut ? bsTimedOut :

  − bsAborted;

  − BuildStepStatus buildStepStatus =

  − result.status == RemoteResult::rrPermanentFailure ? bssFailed :

  − result.status == RemoteResult::rrTimedOut ? bssTimedOut :

  − bssAborted;
  [7.200]

  [7.4841]
  + while (true) {
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1179
  [7.4842]→[7.1564:1658](∅→∅),[7.200]→[7.1564:1658](∅→∅),[7.1658]→[7.4843:4907](∅→∅)
  − /* For regular failures, we don't care about the error

  − message. */

  − if (buildStatus != bsAborted) result.errorMsg = "";
  [7.4842]

  [7.16983]
  + /* Get the builds and steps that depend on this step. */

  + std::set<Build::ptr> indirect;

  + {

  + auto steps_(steps.lock());

  + std::set<Step::ptr> steps;

  + getDependents(step, indirect, steps);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1186
  [7.16984]→[6.576:1192](∅→∅)
  − /* Create failed build steps for every build that depends

  − on this. For cached failures, only create a step for

  − builds that don't have this step as top-level

  − (otherwise the user won't be able to see what caused

  − the build to fail). */

  − for (auto build2 : dependents) {

  − if (build == build2) continue;

  − if (cachedFailure && build2->drvPath == step->drvPath) continue;

  − createBuildStep(txn, 0, build2, step, machine->sshName,

  − buildStepStatus, result.errorMsg, build->id);
  [7.16984]

  [7.2270]
  + /* If there are no builds left, delete all referring

  + steps from ‘steps’. As for the success case, we can

  + be certain no new referrers can be added. */

  + if (indirect.empty()) {

  + for (auto & s : steps) {

  + printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath);

  + nrStepsDone++;

  + steps_->erase(s->drvPath);

  + }

  + break;

  + }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1198
  [7.2284]

  [7.5100]
  +

  + /* Update the database. */

  + {

  + pqxx::work txn(*conn);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1203
  [7.5101]→[7.5101:5297](∅→∅)
  − if (!cachedFailure)

  − finishBuildStep(txn, result.startTime, result.stopTime, build->id,

  − stepNr, machine->sshName, buildStepStatus, result.errorMsg);
  [7.5101]

  [7.2284]
  + BuildStatus buildStatus =

  + result.status == RemoteResult::rrPermanentFailure ? bsFailed :

  + result.status == RemoteResult::rrTimedOut ? bsTimedOut :

  + bsAborted;

  + BuildStepStatus buildStepStatus =

  + result.status == RemoteResult::rrPermanentFailure ? bssFailed :

  + result.status == RemoteResult::rrTimedOut ? bssTimedOut :

  + bssAborted;

  +

  + /* For regular failures, we don't care about the error

  + message. */

  + if (buildStatus != bsAborted) result.errorMsg = "";

  +

  + /* Create failed build steps for every build that depends

  + on this. For cached failures, only create a step for

  + builds that don't have this step as top-level

  + (otherwise the user won't be able to see what caused

  + the build to fail). */

  + for (auto & build2 : indirect) {

  + if (build == build2) continue;

  + if (cachedFailure && build2->drvPath == step->drvPath) continue;

  + createBuildStep(txn, 0, build2, step, machine->sshName,

  + buildStepStatus, result.errorMsg, build->id);

  + }

  +

  + if (!cachedFailure)

  + finishBuildStep(txn, result.startTime, result.stopTime, build->id,

  + stepNr, machine->sshName, buildStepStatus, result.errorMsg);

  +

  + /* Mark all builds that depend on this derivation as failed. */

  + for (auto & build2 : indirect) {

  + printMsg(lvlError, format("marking build %1% as failed") % build2->id);

  + assert(!build->finishedInDB);

  + txn.parameterized

  + ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")

  + (build2->id)

  + ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))

  + (result.startTime)

  + (result.stopTime)

  + (cachedFailure ? 1 : 0).exec();

  + nrBuildsDone++;

  + }

  +

  + /* Remember failed paths in the database so that they

  + won't be built again. */

  + if (!cachedFailure && result.status == RemoteResult::rrPermanentFailure)

  + for (auto & path : outputPaths(step->drv))

  + txn.parameterized("insert into FailedPaths values ($1)")(path).exec();
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1252
  [7.2285]→[7.16984:17060](∅→∅),[7.16984]→[7.16984:17060](∅→∅),[7.17060]→[6.1193:1890](∅→∅)
  − /* Mark all builds that depend on this derivation as failed. */

  − for (auto build2 : dependents) {

  − printMsg(lvlError, format("marking build %1% as failed") % build2->id);

  − txn.parameterized

  − ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")

  − (build2->id)

  − ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))

  − (result.startTime)

  − (result.stopTime)

  − (cachedFailure ? 1 : 0).exec();

  − build2->finishedInDB = true; // FIXME: txn might fail

  − nrBuildsDone++;
  [7.2285]

  [6.1890]
  + txn.commit();
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1255
  [7.2527]→[7.2527:2868](∅→∅)
  − /* Remember failed paths in the database so that they

  − won't be built again. */

  − if (!cachedFailure && result.status == RemoteResult::rrPermanentFailure)

  − for (auto & path : outputPaths(step->drv))

  − txn.parameterized("insert into FailedPaths values ($1)")(path).exec();
  [7.2527]

  [7.17548]
  + /* Remove the indirect dependencies from ‘builds’. This

  + will cause them to be destroyed. */

  + for (auto & b : indirect) {

  + auto builds_(builds.lock());

  + b->finishedInDB = true;

  + builds_->erase(b->id);

  + }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1264
  [7.17559]→[7.17559:17581](∅→∅)
  − txn.commit();
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1265
  [7.17588]→[7.17588:17589](∅→∅),[7.17589]→[7.9140:9391](∅→∅),[7.9391]→[6.1905:2115](∅→∅)
  −

  − /* In case of success, destroy all Build objects of which ‘step’

  − is the top-level derivation. In case of failure, destroy all

  − dependent Build objects. Any Steps not referenced by other

  − Builds will be destroyed as well. */

  − for (auto build2 : dependents)

  − if (build2->toplevel == step || result.status != RemoteResult::rrSuccess) {

  − auto builds_(builds.lock());

  − builds_->erase(build2->id);

  − }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1266
  [7.9570]→[7.9570:9718](∅→∅),[7.9718]→[6.2116:2181](∅→∅),[6.2181]→[7.6378:6379](∅→∅),[7.6378]→[7.6378:6379](∅→∅)
  − /* Remove the step from the graph. In case of success, make

  − dependent build steps runnable if they have no other

  − dependencies. */

  − destroyStep(step, result.status == RemoteResult::rrSuccess);

  −
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1274
  [7.2119]

  [7.14683]
  +

  + assert(!build->finishedInDB);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1304
  [7.19015]→[7.19015:19072](∅→∅)
  − build->finishedInDB = true; // FIXME: txn might fail