hydra-queue-runner: Implement timeouts

Dependencies

• [2] WDNUKCTN Queue monitor: Get only the fields we need
• [3] OCZ4LSGG Automatically retry aborted builds
• [4] LJILHOJ7 Create BuildSteps race-free
• [5] 24BMQDZA Start of single-process hydra-queue-runner
• [6] 5AIYUMTB Basic remote building
• [7] RYTQLATY Keep track of failed paths in the Hydra database
• [8] YZAI5GQU Implement a database connection pool
• [9] 2IQRXLWE Support cancelling builds
• [10] GKZN4UV7 Make the queue monitor more robust, and better debug output
• [11] ENXUSMSV Make concurrency more robust
• [12] 6TY4LNHH Finish copyClosure
• [*] UQQ4IL55 Add a error type for "unsupported system type"
• [*] C6HOMHZW Don't try to handle SIGINT
• [*] IK53RV4V
• [*] U2X3Y53G make sure status images always display for a list of builds
• [*] 6KIJX24R Get rid of unnecessary [%- and -%] tags
• [*] T6KISLR6 Fix indentation
• [*] N22GPKYT * Put info about logs / build products in the DB.
• [*] PMNWRTGJ Add multiple output support

Change contents

• replacement in src/hydra-queue-runner/build-remote.cc at line 112
  [4.2872]→[4.2872:2925](∅→∅)
  − const nix::Path & logDir, RemoteResult & result)
  [4.2872]

  [4.2925]
  + const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,

  + RemoteResult & result)
• replacement in src/hydra-queue-runner/build-remote.cc at line 156
  [4.4251]→[4.4251:4352](∅→∅)
  − writeInt(3600, to); // == maxSilentTime, FIXME

  − writeInt(7200, to); // == buildTimeout, FIXME
  [4.4251]

  [4.4352]
  + writeInt(maxSilentTime, to);

  + writeInt(buildTimeout, to);

  + // FIXME: send maxLogSize.
• replacement in src/hydra-queue-runner/build-remote.hh at line 21
  [4.5729]→[4.5729:5783](∅→∅)
  − const nix::Path & logDir, RemoteResult & result);
  [4.5729]
  + const nix::Path & logDir, unsigned int maxSilentTime, unsigned int buildTimeout,

  + RemoteResult & result);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 46
  [4.5180]

  [14.0]
  + bsTimedOut = 7,
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 55
  [4.5271]

  [14.24]
  + bssTimedOut = 7,
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 82
  [4.171]

  [4.171]
  + unsigned int maxSilentTime, buildTimeout;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 487
  [4.1836]→[2.0:157](∅→∅)
  − auto res = txn.parameterized("select id, project, jobset, job, drvPath from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();
  [4.1836]

  [4.9906]
  + auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 499
  [4.2450]

  [4.10056]
  + build->maxSilentTime = row["maxsilent"].as<int>();

  + build->buildTimeout = row["timeout"].as<int>();

  + std::cerr << build->id << " " << build->buildTimeout << std::endl;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 984
  [4.15294]→[4.1843:2010](∅→∅)
  − printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by %3% builds)")

  − % step->drvPath % machine->sshName % dependents.size());
  [4.15294]

  [4.8135]
  + printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)")

  + % step->drvPath % machine->sshName % build->id % (dependents.size() - 1));
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1014
  [4.999]→[4.999:1108](∅→∅)
  − buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, logDir, result);
  [4.999]

  [4.1108]
  + /* FIXME: referring builds may have conflicting timeouts. */

  + buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv,

  + logDir, build->maxSilentTime, build->buildTimeout, result);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1077
  [3.4613]→[3.4613:4703](∅→∅)
  − result.status == RemoteResult::rrPermanentFailure ? bsFailed : bsAborted;
  [3.4613]

  [3.4703]
  + result.status == RemoteResult::rrPermanentFailure ? bsFailed :

  + result.status == RemoteResult::rrTimedOut ? bsTimedOut :

  + bsAborted;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1081
  [3.4749]→[3.4749:4841](∅→∅)
  − result.status == RemoteResult::rrPermanentFailure ? bssFailed : bssAborted;
  [3.4749]

  [3.4841]
  + result.status == RemoteResult::rrPermanentFailure ? bssFailed :

  + result.status == RemoteResult::rrTimedOut ? bssTimedOut :

  + bssAborted;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1238
  [4.14778]

  [15.40]
  + sleep(5);

  +
• edit in src/root/common.tt at line 207
  [17.688]

  [18.4181]
  + [% ELSIF buildstatus == 7 %]

  + <img src="[% c.uri_for("/static/images/warning_${size}.png") %]" alt="Timed out" class="build-status" />
• edit in src/root/common.tt at line 234
  [19.2353]

  [14.924]
  + [% ELSIF buildstatus == 7 %]

  + <span class="error">Timed out</span>
• edit in src/sql/hydra.sql at line 183
  [21.19738]

  [14.1015]
  + -- 7 = timed out