grahamc/hydra2 - Change BG6PEOB2M2Y56QPVMELU7VNNCGNMSQ2K6ATBUCPJLKPLTDWNJQ5AC

Make the output size limit configurable

The maximum output size per build step (as the sum of the NARs of each output) can be set via hydra.conf, e.g.

max-output-size = 1000000000

The default is 2 GiB.

Also refactored the build error / status handling a bit.

Created by Eelco Dolstra on March 9, 2016

BG6PEOB2M2Y56QPVMELU7VNNCGNMSQ2K6ATBUCPJLKPLTDWNJQ5AC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/build-remote.cc at line 264 [6.85]

B:BD[5.1303] → [5.1303:1509]

            if (res == 100) result.status = BuildResult::PermanentFailure;
            else if (res == 101) result.status = BuildResult::TimedOut;
            else result.status = BuildResult::MiscFailure;

[5.1303]

[5.1509]

            if (res == 100) {
                result.stepStatus = bsFailed;
                result.canCache = true;
            }
            else if (res == 101) {
                result.stepStatus = bsTimedOut;
            }
            else {
                result.stepStatus = bsAborted;
                result.canRetry = true;
            }

Replacement in src/hydra-queue-runner/build-remote.cc at line 277 [6.85]
B:BD[5.1539] → [5.1539:1583]
```
        result.status = BuildResult::Built;
```
[5.1539]
[5.1583]
```
        result.stepStatus = bsSuccess;
```
Deletion in src/hydra-queue-runner/build-remote.cc at line 279 [6.85]
B:BD[5.1596] → [5.1596:1647]
```
        result.status = (BuildResult::Status) res;
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 280 [6.85]

B:BD[5.1691] → [5.1691:1730]

        if (!result.success()) return;

[5.1691]

[6.4785]

        switch ((BuildResult::Status) res) {
            case BuildResult::Built:
                result.stepStatus = bsSuccess;
                break;
            case BuildResult::Substituted:
            case BuildResult::AlreadyValid:
                result.stepStatus = bsSuccess;
                result.isCached = true;
                break;
            case BuildResult::PermanentFailure:
                result.stepStatus = bsFailed;
                result.canCache = true;
                result.errorMsg = "";
                break;
            case BuildResult::InputRejected:
            case BuildResult::OutputRejected:
                result.stepStatus = bsFailed;
                result.canCache = true;
                break;
            case BuildResult::TransientFailure:
                result.stepStatus = bsFailed;
                result.canRetry = true;
                result.errorMsg = "";
                break;
            case BuildResult::CachedFailure: // cached on the build machine
                result.stepStatus = bsCachedFailure;
                result.canCache = true;
                result.errorMsg = "";
                break;
            case BuildResult::TimedOut:
                result.stepStatus = bsTimedOut;
                result.errorMsg = "";
                break;
            case BuildResult::MiscFailure:
                result.stepStatus = bsAborted;
                result.canRetry = true;
                break;
            case BuildResult::LogLimitExceeded:
                result.stepStatus = bsLogLimitExceeded;
                break;
            default:
                result.stepStatus = bsAborted;
                break;
        }
        if (result.stepStatus != bsSuccess) return;

Insertion in src/hydra-queue-runner/build-remote.cc at line 326 [6.85]
[6.4791]
[6.4791]
```
    result.errorMsg = "";
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 331 [6.85]

B:BD[5.1826] → [5.1826:1925]

    if (result.status == BuildResult::Substituted || result.status == BuildResult::AlreadyValid) {

[5.1826]

[2.0]

    if (result.isCached) {

Insertion in src/hydra-queue-runner/build-remote.cc at line 359 [6.85]

[3.524]

        if (totalNarSize > maxOutputSize) {
            result.stepStatus = bsNarSizeLimitExceeded;
            return;
        }

Replacement in src/hydra-queue-runner/builder.cc at line 108 [7.133]

B:BD[7.3711] → [7.3711:3845]

    bool cachedFailure = checkCachedFailure(step, *conn);
    if (cachedFailure)
        result.status = BuildResult::CachedFailure;

[7.3711]

[7.3845]

    if (checkCachedFailure(step, *conn))
        result.stepStatus = bsCachedFailure;

Insertion in src/hydra-queue-runner/builder.cc at line 125 [7.133]

[8.1432]

[7.4474]

        } catch (NoTokens & e) {
            result.stepStatus = bsNarSizeLimitExceeded;

Replacement in src/hydra-queue-runner/builder.cc at line 128 [7.133]

B:BD[7.4504] → [7.4504:4558]

            result.status = BuildResult::MiscFailure;

[7.4504]

[7.4558]

            result.stepStatus = bsAborted;

Replacement in src/hydra-queue-runner/builder.cc at line 132 [7.133]
B:BD[7.4608] → [9.321:351]
```
        if (result.success())
```
[7.4608]
[9.351]
```
        if (result.stepStatus == bsSuccess)
```
Replacement in src/hydra-queue-runner/builder.cc at line 162 [7.133]
B:BD[7.5150] → [7.5150:5179]
```
    if (result.canRetry()) {
```
[7.5150]
[7.5179]
```
    if (result.canRetry) {
```
Replacement in src/hydra-queue-runner/builder.cc at line 181 [7.133]
B:BD[7.5786] → [7.5786:5814]
```
    if (result.success()) {
```
[7.5786]
[7.5814]
```
    if (result.stepStatus == bsSuccess) {
```

Replacement in src/hydra-queue-runner/builder.cc at line 228 [7.133]

B:BD[7.7519] → [7.7519:7622]

                    markSucceededBuild(txn, b, res, build != b || result.status != BuildResult::Built,

[7.7519]

[7.7622]

                    markSucceededBuild(txn, b, res, build != b || result.isCached,

Deletion in src/hydra-queue-runner/builder.cc at line 311 [7.133]

B:BD[7.10398] → [7.10398:10515]

B:BD[7.10515] → [10.482:572]

∅:D[10.572] → [7.10515:10597]

B:BD[7.10515] → [7.10515:10597]


                BuildStatus buildStatus =
                    result.status == BuildResult::TimedOut ? bsTimedOut :
                    result.status == BuildResult::LogLimitExceeded ? bsLogLimitExceeded :
                    result.canRetry() ? bsAborted :
                    bsFailed;

Replacement in src/hydra-queue-runner/builder.cc at line 314 [7.133]

B:BD[7.10910] → [7.10910:11117]

B:BD[7.11117] → [10.665:795]

                if (result.status == BuildResult::PermanentFailure ||
                    result.status == BuildResult::TransientFailure ||
                    result.status == BuildResult::CachedFailure ||
                    result.status == BuildResult::TimedOut ||
                    result.status == BuildResult::LogLimitExceeded)

[7.10910]

[7.11177]

                if (result.stepStatus != bsAborted)

Replacement in src/hydra-queue-runner/builder.cc at line 317 [7.133]

B:BD[7.11220] → [7.11220:11545]

                /* Create failed build steps for every build that depends
                   on this. For cached failures, only create a step for
                   builds that don't have this step as top-level
                   (otherwise the user won't be able to see what caused
                   the build to fail). */

[7.11220]

[7.11545]

                /* Create failed build steps for every build that
                   depends on this, except when this step is cached
                   and is the top-level of that build (since then it's
                   redundant with the build's isCachedBuild field). */

Replacement in src/hydra-queue-runner/builder.cc at line 322 [7.133]

B:BD[7.11594] → [7.11594:11736]

                    if ((cachedFailure && build2->drvPath == step->drvPath) ||
                        (!cachedFailure && build == build2) ||

[7.11594]

[7.11736]

                    if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) ||
                        (result.stepStatus != bsCachedFailure && build == build2) ||

Replacement in src/hydra-queue-runner/builder.cc at line 327 [7.133]

B:BD[7.11892] → [4.243:331]

                        buildStatus, result.errorMsg, build == build2 ? 0 : build->id);

[7.11892]

[7.11984]

                        result.stepStatus, result.errorMsg, build == build2 ? 0 : build->id);

Replacement in src/hydra-queue-runner/builder.cc at line 330 [7.133]

B:BD[7.12003] → [7.12003:12039]

                if (!cachedFailure)

[7.12003]

[11.691]

                if (result.stepStatus != bsCachedFailure)

Replacement in src/hydra-queue-runner/builder.cc at line 332 [7.133]

B:BD[11.784] → [4.332:424]

                        build->id, stepNr, machine->sshName, buildStatus, result.errorMsg);

[11.784]

[7.12211]

                        build->id, stepNr, machine->sshName, result.stepStatus, result.errorMsg);

Replacement in src/hydra-queue-runner/builder.cc at line 341 [7.133]

B:BD[7.12733] → [7.12733:12855]

                        ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))

[7.12733]

[7.12855]

                        ((int) (build2->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus()))

Replacement in src/hydra-queue-runner/builder.cc at line 344 [7.133]

B:BD[7.12940] → [7.12940:12996]

                        (cachedFailure ? 1 : 0).exec();

[7.12940]

[7.12996]

                        (result.stepStatus == bsCachedFailure ? 1 : 0).exec();

Replacement in src/hydra-queue-runner/builder.cc at line 350 [7.133]

B:BD[7.13165] → [7.13165:13251]

                if (!cachedFailure && result.status == BuildResult::PermanentFailure)

[7.13165]

[12.695]

                if (result.stepStatus != bsCachedFailure && result.canCache)

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 44 [14.4840]

[13.560]

[6.7841]

    {
        std::string s = hydraConfig["max-output-size"];
        if (s != "") string2Int(s, maxOutputSize);
    }

Insertion in src/hydra-queue-runner/state.hh at line 36 [15.1128]
[10.825]
[4.996]
```
    bsNarSizeLimitExceeded = 11,
```
Replacement in src/hydra-queue-runner/state.hh at line 41 [15.1128]
B:BD[15.1921] → [5.3621:3660]
```
struct RemoteResult : nix::BuildResult
```
[15.1921]
[16.1573]
```
struct RemoteResult
```

Insertion in src/hydra-queue-runner/state.hh at line 43 [15.1128]

[16.1575]

[16.1747]

    BuildStatus stepStatus = bsAborted;
    bool canRetry = false; // for bsAborted
    bool isCached = false; // for bsSucceed
    bool canCache = false; // for bsFailed
    std::string errorMsg; // for bsAborted

Replacement in src/hydra-queue-runner/state.hh at line 54 [15.1128]
B:BD[5.3662] → [5.3662:3682]
```
    bool canRetry()
```
[5.3662]
[5.3682]
```
    BuildStatus buildStatus()
```

Replacement in src/hydra-queue-runner/state.hh at line 56 [15.1128]

B:BD[5.3688] → [5.3688:3756]

        return status == TransientFailure || status == MiscFailure;

[5.3688]

[5.3756]

        return stepStatus == bsCachedFailure ? bsFailed : stepStatus;

Insertion in src/hydra-queue-runner/state.hh at line 360 [15.1128]
[17.1503]
[15.6778]
```
    size_t maxOutputSize = 2ULL << 30;
```

Insertion in src/root/build.tt at line 68 [18.7350]

[10.963]

[14.20001]

              [% ELSIF step.status == 11 %]
                <span class="error">Output limit exceeded</span>

Insertion in src/root/common.tt at line 210 [20.2803]

[10.1118]

[19.4181]

    [% ELSIF buildstatus == 11 %]
      <img src="[% c.uri_for("/static/images/warning_${size}.png") %]" alt="Output size limit exceeded" class="build-status" />

Insertion in src/root/common.tt at line 241 [20.2803]

[10.1205]

[21.2353]

    [% ELSIF buildstatus == 11 %]
      <span class="error">Output limit exceeded</span>

Insertion in src/sql/hydra.sql at line 195 [23.1]
[10.1238]
[22.4847]
```
    --  11 = NAR size limit exceeded
```