grahamc/hydra2 - Change BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC

Temporarily disable machines on any exception, not just connection failures

Created by Eelco Dolstra on March 22, 2016

BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/build-remote.cc at line 133 [5.85]
∅:D[2.795] → [5.3247:3264]
B:BD[5.3247] → [5.3247:3264]
B:BD[5.3264] → [2.796:847]
```
    Child child;
    openConnection(machine, tmpDir, logFD, child);
```
[2.795]
[5.3315]
```
    try {
```
Replacement in src/hydra-queue-runner/build-remote.cc at line 135 [5.85]
B:BD[5.3316] → [5.3316:3335]
```
    logFD.close();
```
[5.3316]
[5.3335]
```
        Child child;
        openConnection(machine, tmpDir, logFD, child);
```
Replacement in src/hydra-queue-runner/build-remote.cc at line 138 [5.85]
B:BD[5.3336] → [5.3336:3392]
```
    FdSource from(child.from);
    FdSink to(child.to);
```
[5.3336]
[3.23]
```
        logFD.close();
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 140 [5.85]

B:BD[3.24] → [3.24:133]

    Finally updateStats([&]() {
        bytesReceived += from.read;
        bytesSent += to.written;
    });

[3.24]

[5.3392]

        FdSource from(child.from);
        FdSink to(child.to);

Replacement in src/hydra-queue-runner/build-remote.cc at line 143 [5.85]

B:BD[5.3393] → [5.3393:3414]

B:BD[5.3414] → [6.133:165]

B:BD[6.165] → [7.0:32]

    /* Handshake. */
    bool sendDerivation = true;
    unsigned int remoteVersion;

[5.3393]

[7.32]

        Finally updateStats([&]() {
            bytesReceived += from.read;
            bytesSent += to.written;
        });

Replacement in src/hydra-queue-runner/build-remote.cc at line 148 [5.85]
∅:D[7.33] → [8.79:89]
∅:D[6.165] → [8.79:89]
B:BD[5.3414] → [8.79:89]
B:BD[8.89] → [7.34:72]
∅:D[7.72] → [8.172:192]
∅:D[6.221] → [8.172:192]
B:BD[8.172] → [8.172:192]
```
    try {
        to << SERVE_MAGIC_1 << 0x202;
        to.flush();
```
[7.33]
[5.3505]
```
        /* Handshake. */
        bool sendDerivation = true;
        unsigned int remoteVersion;
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 152 [5.85]

B:BD[5.3506] → [8.193:273]

B:BD[8.273] → [9.418:533]

B:BD[9.533] → [7.73:168]

∅:D[7.168] → [9.534:655]

B:BD[8.475] → [9.534:655]

B:BD[9.655] → [7.169:221]

∅:D[10.46] → [6.268:304]

∅:D[11.90] → [6.268:304]

∅:D[7.221] → [6.268:304]

B:BD[6.268] → [6.268:304]

        unsigned int magic = readInt(from);
        if (magic != SERVE_MAGIC_2)
            throw Error(format("protocol mismatch with ‘nix-store --serve’ on ‘%1%’") % machine->sshName);
        remoteVersion = readInt(from);
        if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
            throw Error(format("unsupported ‘nix-store --serve’ protocol version on ‘%1%’") % machine->sshName);
        if (GET_PROTOCOL_MINOR(remoteVersion) >= 1)
            sendDerivation = false;

[5.3506]

[11.91]

        try {
            to << SERVE_MAGIC_1 << 0x202;
            to.flush();
            unsigned int magic = readInt(from);
            if (magic != SERVE_MAGIC_2)
                throw Error(format("protocol mismatch with ‘nix-store --serve’ on ‘%1%’") % machine->sshName);
            remoteVersion = readInt(from);
            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
                throw Error(format("unsupported ‘nix-store --serve’ protocol version on ‘%1%’") % machine->sshName);
            if (GET_PROTOCOL_MINOR(remoteVersion) >= 1)
                sendDerivation = false;

Replacement in src/hydra-queue-runner/build-remote.cc at line 165 [5.85]

∅:D[11.92] → [8.587:647]

∅:D[6.304] → [8.587:647]

∅:D[9.655] → [8.587:647]

B:BD[8.587] → [8.587:647]

    } catch (EndOfFile & e) {
        child.pid.wait(true);

[11.92]

[12.0]

        } catch (EndOfFile & e) {
            child.pid.wait(true);
            string s = chomp(readFile(result.logFile));
            throw Error(format("cannot connect to ‘%1%’: %2%") % machine->sshName % s);
        }

Deletion in src/hydra-queue-runner/build-remote.cc at line 172 [5.85]

B:BD[12.11] → [12.11:323]

            /* Disable this machine until a certain period of time has
               passed. This period increases on every consecutive
               failure. However, don't count failures that occurred
               soon after the last one (to take into account steps
               started in parallel). */

Replacement in src/hydra-queue-runner/build-remote.cc at line 173 [5.85]

B:BD[12.382] → [12.382:998]

            auto now = std::chrono::system_clock::now();
            if (info->consecutiveFailures == 0 || info->lastFailure < now - std::chrono::seconds(30)) {
                info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
                info->lastFailure = now;
                int delta = retryInterval * powf(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
                printMsg(lvlInfo, format("will disable machine ‘%1%’ for %2%s") % machine->sshName % delta);
                info->disabledUntil = now + std::chrono::seconds(delta);
            }

[12.382]

[12.998]

            info->consecutiveFailures = 0;

Replacement in src/hydra-queue-runner/build-remote.cc at line 176 [5.85]

∅:D[12.1009] → [13.354:406]

B:BD[8.647] → [13.354:406]

B:BD[13.406] → [9.656:744]

∅:D[13.485] → [14.0:7]

∅:D[9.744] → [14.0:7]

B:BD[8.749] → [14.0:7]

B:BD[14.7] → [12.1010:1117]

∅:D[12.1117] → [6.305:376]

B:BD[14.7] → [6.305:376]

B:BD[6.376] → [15.0:73]

∅:D[15.73] → [6.446:650]

B:BD[6.446] → [6.446:650]

B:BD[6.650] → [15.74:115]

        string s = chomp(readFile(result.logFile));
        throw Error(format("cannot connect to ‘%1%’: %2%") % machine->sshName % s);
    }
    {
        auto info(machine->state->connectInfo.lock());
        info->consecutiveFailures = 0;
    }
    /* Gather the inputs. If the remote side is Nix <= 1.9, we have to
       copy the entire closure of ‘drvPath’, as well as the required
       outputs of the input derivations. On Nix > 1.9, we only need to
       copy the immediate sources of the derivation and the required
       outputs of the input derivations. */
    PathSet inputs;
    BasicDerivation basicDrv(step->drv);

[12.1009]

[6.650]

        /* Gather the inputs. If the remote side is Nix <= 1.9, we have to
           copy the entire closure of ‘drvPath’, as well as the required
           outputs of the input derivations. On Nix > 1.9, we only need to
           copy the immediate sources of the derivation and the required
           outputs of the input derivations. */
        PathSet inputs;
        BasicDerivation basicDrv(step->drv);

Replacement in src/hydra-queue-runner/build-remote.cc at line 184 [5.85]

B:BD[6.651] → [6.651:797]

    if (sendDerivation)
        inputs.insert(step->drvPath);
    else
        for (auto & p : step->drv.inputSrcs)
            inputs.insert(p);

[6.651]

[6.797]

        if (sendDerivation)
            inputs.insert(step->drvPath);
        else
            for (auto & p : step->drv.inputSrcs)
                inputs.insert(p);

Replacement in src/hydra-queue-runner/build-remote.cc at line 190 [5.85]

∅:D[6.798] → [9.782:829]

B:BD[9.782] → [9.782:829]

∅:D[9.829] → [14.108:252]

B:BD[14.108] → [14.108:252]

B:BD[14.252] → [6.799:893]

B:BD[6.893] → [15.116:171]

    for (auto & input : step->drv.inputDrvs) {
        Derivation drv2 = readDerivation(input.first);
        for (auto & name : input.second) {
            auto i = drv2.outputs.find(name);
            if (i == drv2.outputs.end()) continue;
            inputs.insert(i->second.path);
            basicDrv.inputSrcs.insert(i->second.path);

[6.798]

[14.324]

        for (auto & input : step->drv.inputDrvs) {
            Derivation drv2 = readDerivation(input.first);
            for (auto & name : input.second) {
                auto i = drv2.outputs.find(name);
                if (i == drv2.outputs.end()) continue;
                inputs.insert(i->second.path);
                basicDrv.inputSrcs.insert(i->second.path);
            }

Deletion in src/hydra-queue-runner/build-remote.cc at line 199 [5.85]
∅:D[14.334] → [8.749:755]
B:BD[8.749] → [8.749:755]
```
    }
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 200 [5.85]

B:BD[16.592] → [16.592:861]

    /* Ensure that the inputs exist in the destination store. This is
       a no-op for regular stores, but for the binary cache store,
       this will copy the inputs to the binary cache from the local
       store. */
    destStore->buildPaths(basicDrv.inputSrcs);

[16.592]

[5.3876]

        /* Ensure that the inputs exist in the destination store. This is
           a no-op for regular stores, but for the binary cache store,
           this will copy the inputs to the binary cache from the local
           store. */
        destStore->buildPaths(basicDrv.inputSrcs);

Replacement in src/hydra-queue-runner/build-remote.cc at line 206 [5.85]

B:BD[5.3877] → [5.3877:3911]

B:BD[5.3911] → [16.862:916]

∅:D[16.916] → [17.0:68]

B:BD[9.873] → [17.0:68]

∅:D[17.68] → [18.26:98]

B:BD[19.205] → [18.26:98]

B:BD[18.98] → [17.69:251]

    /* Copy the input closure. */
    if (/* machine->sshName != "localhost" */ true) {
        auto mc1 = std::make_shared<MaintainCount>(nrStepsWaiting);
        std::lock_guard<std::mutex> sendLock(machine->state->sendLock);
        mc1.reset();
        MaintainCount mc2(nrStepsCopyingTo);
        printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName);

[5.3877]

[20.0]

        /* Copy the input closure. */
        if (/* machine->sshName != "localhost" */ true) {
            auto mc1 = std::make_shared<MaintainCount>(nrStepsWaiting);
            std::lock_guard<std::mutex> sendLock(machine->state->sendLock);
            mc1.reset();
            MaintainCount mc2(nrStepsCopyingTo);
            printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName);

Replacement in src/hydra-queue-runner/build-remote.cc at line 214 [5.85]

B:BD[20.1] → [20.1:55]

        auto now1 = std::chrono::steady_clock::now();

[20.1]

[20.55]

            auto now1 = std::chrono::steady_clock::now();

Replacement in src/hydra-queue-runner/build-remote.cc at line 216 [5.85]

B:BD[20.56] → [3.134:192]

        copyClosureTo(destStore, from, to, inputs, true);

[20.56]

[20.57]

            copyClosureTo(destStore, from, to, inputs, true);

Replacement in src/hydra-queue-runner/build-remote.cc at line 218 [5.85]

B:BD[20.58] → [20.58:112]

        auto now2 = std::chrono::steady_clock::now();

[20.58]

[20.112]

            auto now2 = std::chrono::steady_clock::now();

Replacement in src/hydra-queue-runner/build-remote.cc at line 220 [5.85]
B:BD[20.113] → [20.113:216]
∅:D[18.157] → [19.277:283]
∅:D[20.216] → [19.277:283]
∅:D[22.678] → [19.277:283]
∅:D[16.980] → [19.277:283]
B:BD[19.277] → [19.277:283]
```
        result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
    }
```
[20.113]
[5.4064]
```
            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
        }
```
Replacement in src/hydra-queue-runner/build-remote.cc at line 223 [5.85]
B:BD[5.4065] → [13.486:511]
```
    autoDelete.cancel();
```
[5.4065]
[13.511]
```
        autoDelete.cancel();
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 225 [5.85]

∅:D[13.512] → [5.4065:4089]

B:BD[5.4065] → [5.4065:4089]

B:BD[5.4089] → [9.990:1092]

    /* Do the build. */
    printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName);

[13.512]

[6.894]

        /* Do the build. */
        printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName);

Replacement in src/hydra-queue-runner/build-remote.cc at line 228 [5.85]

B:BD[6.895] → [6.895:919]

B:BD[6.919] → [7.222:279]

∅:D[7.279] → [6.1009:1018]

B:BD[6.1009] → [6.1009:1018]

B:BD[6.1018] → [7.280:481]

∅:D[23.209] → [5.4352:4368]

∅:D[7.481] → [5.4352:4368]

∅:D[6.1150] → [5.4352:4368]

B:BD[5.4352] → [5.4352:4368]

    if (sendDerivation)
        to << cmdBuildPaths << PathSet({step->drvPath});
    else
        to << cmdBuildDerivation << step->drvPath << basicDrv;
    to << maxSilentTime << buildTimeout;
    if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
        to << 64 * 1024 * 1024; // == maxLogSize
    to.flush();

[6.895]

[6.1151]

        if (sendDerivation)
            to << cmdBuildPaths << PathSet({step->drvPath});
        else
            to << cmdBuildDerivation << step->drvPath << basicDrv;
        to << maxSilentTime << buildTimeout;
        if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
            to << 64 * 1024 * 1024; // == maxLogSize
        to.flush();

Replacement in src/hydra-queue-runner/build-remote.cc at line 237 [5.85]

∅:D[6.1152] → [5.4368:4400]

B:BD[5.4368] → [5.4368:4400]

B:BD[5.4400] → [24.55:152]

∅:D[24.152] → [5.4429:4460]

B:BD[5.4429] → [5.4429:4460]

    result.startTime = time(0);
    int res;
    {
        MaintainCount mc(nrStepsBuilding);
        res = readInt(from);
    }
    result.stopTime = time(0);

[6.1152]

[6.1153]

        result.startTime = time(0);
        int res;
        {
            MaintainCount mc(nrStepsBuilding);
            res = readInt(from);
        }
        result.stopTime = time(0);

Replacement in src/hydra-queue-runner/build-remote.cc at line 245 [5.85]

B:BD[6.1154] → [6.1154:1303]

B:BD[6.1303] → [4.0:116]

    if (sendDerivation) {
        if (res) {
            result.errorMsg = (format("%1% on ‘%2%’") % readString(from) % machine->sshName).str();
            if (res == 100) {
                result.stepStatus = bsFailed;
                result.canCache = true;

[6.1154]

[4.116]

        if (sendDerivation) {
            if (res) {
                result.errorMsg = (format("%1% on ‘%2%’") % readString(from) % machine->sshName).str();
                if (res == 100) {
                    result.stepStatus = bsFailed;
                    result.canCache = true;
                }
                else if (res == 101) {
                    result.stepStatus = bsTimedOut;
                }
                else {
                    result.stepStatus = bsAborted;
                    result.canRetry = true;
                }
                return;

Replacement in src/hydra-queue-runner/build-remote.cc at line 261 [5.85]

B:BD[4.130] → [4.130:333]

            else if (res == 101) {
                result.stepStatus = bsTimedOut;
            }
            else {
                result.stepStatus = bsAborted;
                result.canRetry = true;

[4.130]

[4.333]

            result.stepStatus = bsSuccess;
        } else {
            result.errorMsg = readString(from);
            switch ((BuildResult::Status) res) {
                case BuildResult::Built:
                    result.stepStatus = bsSuccess;
                    break;
                case BuildResult::Substituted:
                case BuildResult::AlreadyValid:
                    result.stepStatus = bsSuccess;
                    result.isCached = true;
                    break;
                case BuildResult::PermanentFailure:
                    result.stepStatus = bsFailed;
                    result.canCache = true;
                    result.errorMsg = "";
                    break;
                case BuildResult::InputRejected:
                case BuildResult::OutputRejected:
                    result.stepStatus = bsFailed;
                    result.canCache = true;
                    break;
                case BuildResult::TransientFailure:
                    result.stepStatus = bsFailed;
                    result.canRetry = true;
                    result.errorMsg = "";
                    break;
                case BuildResult::CachedFailure: // cached on the build machine
                    result.stepStatus = bsCachedFailure;
                    result.canCache = true;
                    result.errorMsg = "";
                    break;
                case BuildResult::TimedOut:
                    result.stepStatus = bsTimedOut;
                    result.errorMsg = "";
                    break;
                case BuildResult::MiscFailure:
                    result.stepStatus = bsAborted;
                    result.canRetry = true;
                    break;
                case BuildResult::LogLimitExceeded:
                    result.stepStatus = bsLogLimitExceeded;
                    break;
                default:
                    result.stepStatus = bsAborted;
                    break;

Replacement in src/hydra-queue-runner/build-remote.cc at line 308 [5.85]
∅:D[4.347] → [6.1509:1529]
B:BD[6.1509] → [6.1509:1529]
```
            return;
```
[4.347]
[6.1529]
```
            if (result.stepStatus != bsSuccess) return;
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 310 [5.85]

B:BD[6.1539] → [4.348:387]

∅:D[4.387] → [6.1583:1596]

B:BD[6.1583] → [6.1583:1596]

B:BD[6.1647] → [6.1647:1691]

B:BD[6.1691] → [4.388:2077]

        result.stepStatus = bsSuccess;
    } else {
        result.errorMsg = readString(from);
        switch ((BuildResult::Status) res) {
            case BuildResult::Built:
                result.stepStatus = bsSuccess;
                break;
            case BuildResult::Substituted:
            case BuildResult::AlreadyValid:
                result.stepStatus = bsSuccess;
                result.isCached = true;
                break;
            case BuildResult::PermanentFailure:
                result.stepStatus = bsFailed;
                result.canCache = true;
                result.errorMsg = "";
                break;
            case BuildResult::InputRejected:
            case BuildResult::OutputRejected:
                result.stepStatus = bsFailed;
                result.canCache = true;
                break;
            case BuildResult::TransientFailure:
                result.stepStatus = bsFailed;
                result.canRetry = true;
                result.errorMsg = "";
                break;
            case BuildResult::CachedFailure: // cached on the build machine
                result.stepStatus = bsCachedFailure;
                result.canCache = true;
                result.errorMsg = "";
                break;
            case BuildResult::TimedOut:
                result.stepStatus = bsTimedOut;
                result.errorMsg = "";
                break;
            case BuildResult::MiscFailure:
                result.stepStatus = bsAborted;
                result.canRetry = true;
                break;
            case BuildResult::LogLimitExceeded:
                result.stepStatus = bsLogLimitExceeded;
                break;
            default:
                result.stepStatus = bsAborted;
                break;

[6.1539]

[4.2077]


        result.errorMsg = "";
        /* If the path was substituted or already valid, then we didn't
           get a build log. */
        if (result.isCached) {
            printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName);
            unlink(result.logFile.c_str());
            result.logFile = "";

Deletion in src/hydra-queue-runner/build-remote.cc at line 320 [5.85]
B:BD[4.2087] → [4.2087:2139]
∅:D[6.1730] → [5.4785:4791]
∅:D[4.2139] → [5.4785:4791]
B:BD[5.4785] → [5.4785:4791]
```
        if (result.stepStatus != bsSuccess) return;
    }
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 321 [5.85]

B:BD[4.2141] → [4.2141:2167]

    result.errorMsg = "";

[4.2141]

[5.4791]

        /* Copy the output paths. */
        if (/* machine->sshName != "localhost" */ true) {
            MaintainCount mc(nrStepsCopyingFrom);

Replacement in src/hydra-queue-runner/build-remote.cc at line 325 [5.85]

B:BD[5.4792] → [6.1731:1826]

B:BD[6.1826] → [4.2168:2195]

∅:D[4.2195] → [25.0:136]

B:BD[6.1925] → [25.0:136]

∅:D[25.136] → [6.1925:2000]

B:BD[6.1925] → [6.1925:2000]

    /* If the path was substituted or already valid, then we didn't
       get a build log. */
    if (result.isCached) {
        printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName);
        unlink(result.logFile.c_str());
        result.logFile = "";
    }

[5.4792]

[6.2000]

            auto now1 = std::chrono::steady_clock::now();

Replacement in src/hydra-queue-runner/build-remote.cc at line 327 [5.85]

∅:D[6.2001] → [5.4792:4825]

B:BD[5.4792] → [5.4792:4825]

B:BD[5.4825] → [16.981:1035]

B:BD[16.1035] → [26.0:46]

    /* Copy the output paths. */
    if (/* machine->sshName != "localhost" */ true) {
        MaintainCount mc(nrStepsCopyingFrom);

[6.2001]

[26.46]

            PathSet outputs;
            for (auto & output : step->drv.outputs)
                outputs.insert(output.second.path);

Replacement in src/hydra-queue-runner/build-remote.cc at line 331 [5.85]

B:BD[26.47] → [26.47:101]

        auto now1 = std::chrono::steady_clock::now();

[26.47]

[26.101]

            /* Query the size of the output paths. */
            size_t totalNarSize = 0;
            to << cmdQueryPathInfos << outputs;
            to.flush();
            while (true) {
                if (readString(from) == "") break;
                readString(from); // deriver
                readStrings<PathSet>(from); // references
                readLongLong(from); // download size
                totalNarSize += readLongLong(from);
            }

Replacement in src/hydra-queue-runner/build-remote.cc at line 343 [5.85]

∅:D[26.102] → [22.816:841]

∅:D[9.1404] → [22.816:841]

B:BD[22.816] → [22.816:841]

B:BD[22.841] → [9.1405:1453]

∅:D[9.1453] → [22.883:931]

B:BD[22.883] → [22.883:931]

        PathSet outputs;
        for (auto & output : step->drv.outputs)
            outputs.insert(output.second.path);

[26.102]

[26.103]

            if (totalNarSize > maxOutputSize) {
                result.stepStatus = bsNarSizeLimitExceeded;
                return;
            }

Replacement in src/hydra-queue-runner/build-remote.cc at line 348 [5.85]

B:BD[26.104] → [26.104:523]

        /* Query the size of the output paths. */
        size_t totalNarSize = 0;
        to << cmdQueryPathInfos << outputs;
        to.flush();
        while (true) {
            if (readString(from) == "") break;
            readString(from); // deriver
            readStrings<PathSet>(from); // references
            readLongLong(from); // download size
            totalNarSize += readLongLong(from);
        }

[26.104]

[26.523]

            printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)")
                % step->drvPath % machine->sshName % totalNarSize);

Replacement in src/hydra-queue-runner/build-remote.cc at line 351 [5.85]

B:BD[26.524] → [4.2196:2326]

        if (totalNarSize > maxOutputSize) {
            result.stepStatus = bsNarSizeLimitExceeded;
            return;
        }

[26.524]

[4.2326]

            /* Block until we have the required amount of memory
               available. FIXME: only need this for binary cache
               destination stores. */
            auto resStart = std::chrono::steady_clock::now();
            auto memoryReservation(memoryTokens.get(totalNarSize));
            auto resStop = std::chrono::steady_clock::now();

Replacement in src/hydra-queue-runner/build-remote.cc at line 358 [5.85]

∅:D[4.2327] → [26.524:678]

B:BD[26.524] → [26.524:678]

        printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)")
            % step->drvPath % machine->sshName % totalNarSize);

[4.2327]

[26.678]

            auto resMs = std::chrono::duration_cast<std::chrono::milliseconds>(resStop - resStart).count();
            if (resMs >= 1000)
                printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s")
                    % resMs % totalNarSize % step->drvPath);

Replacement in src/hydra-queue-runner/build-remote.cc at line 363 [5.85]

B:BD[26.679] → [26.679:1014]

        /* Block until we have the required amount of memory
           available. FIXME: only need this for binary cache
           destination stores. */
        auto resStart = std::chrono::steady_clock::now();
        auto memoryReservation(memoryTokens.get(totalNarSize));
        auto resStop = std::chrono::steady_clock::now();

[26.679]

[20.217]

            result.accessor = destStore->getFSAccessor();

Replacement in src/hydra-queue-runner/build-remote.cc at line 365 [5.85]

B:BD[20.218] → [26.1015:1299]

        auto resMs = std::chrono::duration_cast<std::chrono::milliseconds>(resStop - resStart).count();
        if (resMs >= 1000)
            printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s")
                % resMs % totalNarSize % step->drvPath);

[20.218]

[27.231]

            to << cmdExportPaths << 0 << outputs;
            to.flush();
            destStore->importPaths(false, from, result.accessor);

Replacement in src/hydra-queue-runner/build-remote.cc at line 369 [5.85]

B:BD[27.232] → [26.1300:1354]

        result.accessor = destStore->getFSAccessor();

[27.232]

[20.272]

            auto now2 = std::chrono::steady_clock::now();

Replacement in src/hydra-queue-runner/build-remote.cc at line 371 [5.85]

B:BD[20.273] → [3.193:321]

        to << cmdExportPaths << 0 << outputs;
        to.flush();
        destStore->importPaths(false, from, result.accessor);

[20.273]

[20.274]

            result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
        }

Replacement in src/hydra-queue-runner/build-remote.cc at line 374 [5.85]

B:BD[20.275] → [20.275:329]

        auto now2 = std::chrono::steady_clock::now();

[20.275]

[20.329]

        /* Shut down the connection. */
        child.to.close();
        child.pid.wait(true);

Replacement in src/hydra-queue-runner/build-remote.cc at line 378 [5.85]

B:BD[20.330] → [20.330:433]

        result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

[20.330]

[19.387]

    } catch (Error & e) {
        /* Disable this machine until a certain period of time has
           passed. This period increases on every consecutive
           failure. However, don't count failures that occurred soon
           after the last one (to take into account steps started in
           parallel). */
        auto info(machine->state->connectInfo.lock());
        auto now = std::chrono::system_clock::now();
        if (info->consecutiveFailures == 0 || info->lastFailure < now - std::chrono::seconds(30)) {
            info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4);
            info->lastFailure = now;
            int delta = retryInterval * powf(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30);
            printMsg(lvlInfo, format("will disable machine ‘%1%’ for %2%s") % machine->sshName % delta);
            info->disabledUntil = now + std::chrono::seconds(delta);
        }
        throw;

Deletion in src/hydra-queue-runner/build-remote.cc at line 395 [5.85]
∅:D[19.393] → [5.5074:5159]
B:BD[5.5074] → [5.5074:5159]
```
    /* Shut down the connection. */
    child.to.close();
    child.pid.wait(true);
```