Make the output size limit configurable

Dependencies

• [2] TDSBTZKX Add log message
• [3] SL3WSRAC hydra-queue-runner: Limit memory usage
• [4] UNVMKJV5 Unify build and step status codes
• [5] BAFICF73 Support hydra-build-products on binary cache stores
• [6] TPNHTE5V Remove obsolete Builds columns and provide accurate "Running builds"
• [7] UVNTWTWG Prevent download of NARs we just uploaded
• [8] MHVIT4JY Split hydra-queue-runner.cc more
• [9] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [10] FITVNQ2S Keep track of the time we spend copying to/from build machines
• [11] NKQOEVVP Get rid of "will retry" messages after "maybe cancelling..."
• [12] BRAESISH Warn if PostgreSQL appears stalled
• [13] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
• [14] IK2UBDAU Revive jobset scheduling
• [15] HH3LID6L Re-implement log size limits
• [16] 5AIYUMTB Basic remote building
• [17] DKJFD6JN Process Nix API changes
• [18] HJOEIMLR Refactor
• [19] N4IROACV Move buildRemote() into State
• [20] UYUVQWXQ Fix hydra-queue-runner --build-one
• [*] 24BMQDZA Start of single-process hydra-queue-runner
• [*] V2UCCYN3 hydra-queue-runner: Get store mode configuration from hydra.conf
• [*] EOO4EFWD Use a single BinaryCacheStore for all threads
• [*] J5UVLXOK * Start of a basic Catalyst web interface.
• [*] IK53RV4V
• [*] 6KIJX24R Get rid of unnecessary [%- and -%] tags
• [*] T6KISLR6 Fix indentation
• [*] N22GPKYT * Put info about logs / build products in the DB.
• [*] BD3GRK4B * Get rid of "positive failures" and separate log phases.

Change contents

• replacement in src/hydra-queue-runner/build-remote.cc at line 264
  [5.1303]→[5.1303:1509](∅→∅)
  − if (res == 100) result.status = BuildResult::PermanentFailure;

  − else if (res == 101) result.status = BuildResult::TimedOut;

  − else result.status = BuildResult::MiscFailure;
  [5.1303]

  [5.1509]
  + if (res == 100) {

  + result.stepStatus = bsFailed;

  + result.canCache = true;

  + }

  + else if (res == 101) {

  + result.stepStatus = bsTimedOut;

  + }

  + else {

  + result.stepStatus = bsAborted;

  + result.canRetry = true;

  + }
• replacement in src/hydra-queue-runner/build-remote.cc at line 277
  [5.1539]→[5.1539:1583](∅→∅)
  − result.status = BuildResult::Built;
  [5.1539]

  [5.1583]
  + result.stepStatus = bsSuccess;
• edit in src/hydra-queue-runner/build-remote.cc at line 279
  [5.1596]→[5.1596:1647](∅→∅)
  − result.status = (BuildResult::Status) res;
• replacement in src/hydra-queue-runner/build-remote.cc at line 280
  [5.1691]→[5.1691:1730](∅→∅)
  − if (!result.success()) return;
  [5.1691]

  [5.4785]
  + switch ((BuildResult::Status) res) {

  + case BuildResult::Built:

  + result.stepStatus = bsSuccess;

  + break;

  + case BuildResult::Substituted:

  + case BuildResult::AlreadyValid:

  + result.stepStatus = bsSuccess;

  + result.isCached = true;

  + break;

  + case BuildResult::PermanentFailure:

  + result.stepStatus = bsFailed;

  + result.canCache = true;

  + result.errorMsg = "";

  + break;

  + case BuildResult::InputRejected:

  + case BuildResult::OutputRejected:

  + result.stepStatus = bsFailed;

  + result.canCache = true;

  + break;

  + case BuildResult::TransientFailure:

  + result.stepStatus = bsFailed;

  + result.canRetry = true;

  + result.errorMsg = "";

  + break;

  + case BuildResult::CachedFailure: // cached on the build machine

  + result.stepStatus = bsCachedFailure;

  + result.canCache = true;

  + result.errorMsg = "";

  + break;

  + case BuildResult::TimedOut:

  + result.stepStatus = bsTimedOut;

  + result.errorMsg = "";

  + break;

  + case BuildResult::MiscFailure:

  + result.stepStatus = bsAborted;

  + result.canRetry = true;

  + break;

  + case BuildResult::LogLimitExceeded:

  + result.stepStatus = bsLogLimitExceeded;

  + break;

  + default:

  + result.stepStatus = bsAborted;

  + break;

  + }

  + if (result.stepStatus != bsSuccess) return;
• edit in src/hydra-queue-runner/build-remote.cc at line 326
  [5.4791]

  [5.4791]
  +

  + result.errorMsg = "";
• replacement in src/hydra-queue-runner/build-remote.cc at line 331
  [5.1826]→[5.1826:1925](∅→∅)
  − if (result.status == BuildResult::Substituted || result.status == BuildResult::AlreadyValid) {
  [5.1826]

  [2.0]
  + if (result.isCached) {
• edit in src/hydra-queue-runner/build-remote.cc at line 359
  [3.524]

  [3.524]
  + if (totalNarSize > maxOutputSize) {

  + result.stepStatus = bsNarSizeLimitExceeded;

  + return;

  + }

  +
• replacement in src/hydra-queue-runner/builder.cc at line 108
  [5.3711]→[5.3711:3845](∅→∅)
  − bool cachedFailure = checkCachedFailure(step, *conn);

  −

  − if (cachedFailure)

  − result.status = BuildResult::CachedFailure;
  [5.3711]

  [5.3845]
  + if (checkCachedFailure(step, *conn))

  + result.stepStatus = bsCachedFailure;
• edit in src/hydra-queue-runner/builder.cc at line 125
  [5.1432]

  [5.4474]
  + } catch (NoTokens & e) {

  + result.stepStatus = bsNarSizeLimitExceeded;
• replacement in src/hydra-queue-runner/builder.cc at line 128
  [5.4504]→[5.4504:4558](∅→∅)
  − result.status = BuildResult::MiscFailure;
  [5.4504]

  [5.4558]
  + result.stepStatus = bsAborted;
• replacement in src/hydra-queue-runner/builder.cc at line 132
  [5.4608]→[5.321:351](∅→∅)
  − if (result.success())
  [5.4608]

  [5.351]
  + if (result.stepStatus == bsSuccess)
• replacement in src/hydra-queue-runner/builder.cc at line 162
  [5.5150]→[5.5150:5179](∅→∅)
  − if (result.canRetry()) {
  [5.5150]

  [5.5179]
  + if (result.canRetry) {
• replacement in src/hydra-queue-runner/builder.cc at line 181
  [5.5786]→[5.5786:5814](∅→∅)
  − if (result.success()) {
  [5.5786]

  [5.5814]
  + if (result.stepStatus == bsSuccess) {
• replacement in src/hydra-queue-runner/builder.cc at line 228
  [5.7519]→[5.7519:7622](∅→∅)
  − markSucceededBuild(txn, b, res, build != b || result.status != BuildResult::Built,
  [5.7519]

  [5.7622]
  + markSucceededBuild(txn, b, res, build != b || result.isCached,
• edit in src/hydra-queue-runner/builder.cc at line 311
  [5.10398]→[5.10398:10515](∅→∅),[5.10515]→[5.482:572](∅→∅),[5.572]→[5.10515:10597](∅→∅),[5.10515]→[5.10515:10597](∅→∅)
  −

  − BuildStatus buildStatus =

  − result.status == BuildResult::TimedOut ? bsTimedOut :

  − result.status == BuildResult::LogLimitExceeded ? bsLogLimitExceeded :

  − result.canRetry() ? bsAborted :

  − bsFailed;
• replacement in src/hydra-queue-runner/builder.cc at line 314
  [5.10910]→[5.10910:11117](∅→∅),[5.11117]→[5.665:795](∅→∅)
  − if (result.status == BuildResult::PermanentFailure ||

  − result.status == BuildResult::TransientFailure ||

  − result.status == BuildResult::CachedFailure ||

  − result.status == BuildResult::TimedOut ||

  − result.status == BuildResult::LogLimitExceeded)
  [5.10910]

  [5.11177]
  + if (result.stepStatus != bsAborted)
• replacement in src/hydra-queue-runner/builder.cc at line 317
  [5.11220]→[5.11220:11545](∅→∅)
  − /* Create failed build steps for every build that depends

  − on this. For cached failures, only create a step for

  − builds that don't have this step as top-level

  − (otherwise the user won't be able to see what caused

  − the build to fail). */
  [5.11220]

  [5.11545]
  + /* Create failed build steps for every build that

  + depends on this, except when this step is cached

  + and is the top-level of that build (since then it's

  + redundant with the build's isCachedBuild field). */
• replacement in src/hydra-queue-runner/builder.cc at line 322
  [5.11594]→[5.11594:11736](∅→∅)
  − if ((cachedFailure && build2->drvPath == step->drvPath) ||

  − (!cachedFailure && build == build2) ||
  [5.11594]

  [5.11736]
  + if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) ||

  + (result.stepStatus != bsCachedFailure && build == build2) ||
• replacement in src/hydra-queue-runner/builder.cc at line 327
  [5.11892]→[4.243:331](∅→∅)
  − buildStatus, result.errorMsg, build == build2 ? 0 : build->id);
  [5.11892]

  [5.11984]
  + result.stepStatus, result.errorMsg, build == build2 ? 0 : build->id);
• replacement in src/hydra-queue-runner/builder.cc at line 330
  [5.12003]→[5.12003:12039](∅→∅)
  − if (!cachedFailure)
  [5.12003]

  [5.691]
  + if (result.stepStatus != bsCachedFailure)
• replacement in src/hydra-queue-runner/builder.cc at line 332
  [5.784]→[4.332:424](∅→∅)
  − build->id, stepNr, machine->sshName, buildStatus, result.errorMsg);
  [5.784]

  [5.12211]
  + build->id, stepNr, machine->sshName, result.stepStatus, result.errorMsg);
• replacement in src/hydra-queue-runner/builder.cc at line 341
  [5.12733]→[5.12733:12855](∅→∅)
  − ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))
  [5.12733]

  [5.12855]
  + ((int) (build2->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus()))
• replacement in src/hydra-queue-runner/builder.cc at line 344
  [5.12940]→[5.12940:12996](∅→∅)
  − (cachedFailure ? 1 : 0).exec();
  [5.12940]

  [5.12996]
  + (result.stepStatus == bsCachedFailure ? 1 : 0).exec();
• replacement in src/hydra-queue-runner/builder.cc at line 350
  [5.13165]→[5.13165:13251](∅→∅)
  − if (!cachedFailure && result.status == BuildResult::PermanentFailure)
  [5.13165]

  [5.695]
  + if (result.stepStatus != bsCachedFailure && result.canCache)
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 44
  [23.560]

  [5.7841]
  + {

  + std::string s = hydraConfig["max-output-size"];

  + if (s != "") string2Int(s, maxOutputSize);

  + }

  +
• edit in src/hydra-queue-runner/state.hh at line 36
  [5.825]

  [4.996]
  + bsNarSizeLimitExceeded = 11,
• replacement in src/hydra-queue-runner/state.hh at line 41
  [5.1921]→[5.3621:3660](∅→∅)
  − struct RemoteResult : nix::BuildResult
  [5.1921]

  [5.1573]
  + struct RemoteResult
• edit in src/hydra-queue-runner/state.hh at line 43
  [5.1575]

  [5.1747]
  + BuildStatus stepStatus = bsAborted;

  + bool canRetry = false; // for bsAborted

  + bool isCached = false; // for bsSucceed

  + bool canCache = false; // for bsFailed

  + std::string errorMsg; // for bsAborted

  +
• replacement in src/hydra-queue-runner/state.hh at line 54
  [5.3662]→[5.3662:3682](∅→∅)
  − bool canRetry()
  [5.3662]

  [5.3682]
  + BuildStatus buildStatus()
• replacement in src/hydra-queue-runner/state.hh at line 56
  [5.3688]→[5.3688:3756](∅→∅)
  − return status == TransientFailure || status == MiscFailure;
  [5.3688]

  [5.3756]
  + return stepStatus == bsCachedFailure ? bsFailed : stepStatus;
• edit in src/hydra-queue-runner/state.hh at line 360
  [24.1503]

  [5.6778]
  + size_t maxOutputSize = 2ULL << 30;

  +
• edit in src/root/build.tt at line 68
  [5.963]

  [22.20001]
  + [% ELSIF step.status == 11 %]

  + <span class="error">Output limit exceeded</span>
• edit in src/root/common.tt at line 210
  [5.1118]

  [27.4181]
  + [% ELSIF buildstatus == 11 %]

  + <img src="[% c.uri_for("/static/images/warning_${size}.png") %]" alt="Output size limit exceeded" class="build-status" />
• edit in src/root/common.tt at line 241
  [5.1205]

  [28.2353]
  + [% ELSIF buildstatus == 11 %]

  + <span class="error">Output limit exceeded</span>
• edit in src/sql/hydra.sql at line 195
  [5.1238]

  [30.4847]
  + -- 11 = NAR size limit exceeded