Periodically clear orphaned build steps

Dependencies

• [2] RGCPOSHY hydra-queue-runner: Ensure regular status dumps
• [*] MHVIT4JY Split hydra-queue-runner.cc more
• [*] BRAESISH Warn if PostgreSQL appears stalled
• [*] 24BMQDZA Start of single-process hydra-queue-runner
• [*] IE2PRAQU hydra-queue-runner: Send build notifications
• [*] HJOEIMLR Refactor

Change contents

• edit in src/hydra-queue-runner/builder.cc at line 5
  [4.199]

  [4.199]
  + #include "finally.hh"
• edit in src/hydra-queue-runner/builder.cc at line 104
  [4.3557]

  [4.3557]
  + bool stepFinished = false;

  +

  + Finally clearStep([&]() {

  + if (stepNr && !stepFinished) {

  + auto orphanedSteps_(orphanedSteps.lock());

  + orphanedSteps_->emplace(build->id, stepNr);

  + }

  + });
• edit in src/hydra-queue-runner/builder.cc at line 182
  [5.79]

  [4.5501]
  + {
• edit in src/hydra-queue-runner/builder.cc at line 187
  [4.5713]

  [4.5713]
  + }

  + stepFinished = true;
• edit in src/hydra-queue-runner/builder.cc at line 248
  [4.7727]

  [4.7727]
  +

  + stepFinished = true;
• edit in src/hydra-queue-runner/builder.cc at line 373
  [4.13454]

  [4.13454]
  +

  + stepFinished = true;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 828
  [7.2824]

  [7.2824]
  + /* Periodically clean up orphaned busy steps in the database. */

  + std::thread([&]() {

  + while (true) {

  + sleep(180);

  +

  + std::set<std::pair<BuildID, int>> steps;

  + {

  + auto orphanedSteps_(orphanedSteps.lock());

  + if (orphanedSteps_->empty()) continue;

  + steps = *orphanedSteps_;

  + orphanedSteps_->clear();

  + }

  +

  + try {

  + auto conn(dbPool.get());

  + pqxx::work txn(*conn);

  + for (auto & step : steps) {

  + printMsg(lvlError, format("cleaning orphaned step %d of build %d") % step.second % step.first);

  + txn.parameterized

  + ("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy = 1")

  + ((int) bsAborted)

  + (step.first)

  + (step.second).exec();

  + }

  + txn.commit();

  + } catch (std::exception & e) {

  + printMsg(lvlError, format("cleanup thread: %1%") % e.what());

  + auto orphanedSteps_(orphanedSteps.lock());

  + orphanedSteps_->insert(steps.begin(), steps.end());

  + }

  + }

  + }).detach();

  +
• edit in src/hydra-queue-runner/state.hh at line 365
  [2.357]

  [8.6778]
  + /* Steps that were busy while we encounted a PostgreSQL

  + error. These need to be cleared at a later time to prevent them

  + from showing up as busy until the queue runner is restarted. */

  + nix::Sync<std::set<std::pair<BuildID, int>>> orphanedSteps;

  +