Remove the `sendDerivation` logic from the builder

[?]
Feb 23, 2021, 8:50 AM
TJWMCLXXJY7ZZTNCM2B5KAINSQN46NECSXFFF4Y7XIVZKB3TO7SAC

Dependencies

  • [2] J5YPMWCJ Get exact build start/stop times from the remote
  • [3] FNWNZYKU Fix localhost builds
  • [4] EFWDY2V7 Support testing build determinism
  • [5] YVFKZFAP Fix build
  • [6] EVSK5TBB Fix version test
  • [7] G7KWXSFM Distinguish build step states
  • [8] VZKB5CIE Workaround for RemoteStore not supporting cmdBuildDerivation yet
  • [9] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
  • [10] H46LUUZY hydra-queue-runner: don't try to distribute builds on localhost
  • [11] EPWEMRI2 Allow determinism checking for entire jobsets
  • [12] WV4SSAIY Build against nix-master
  • [13] BG6PEOB2 Make the output size limit configurable
  • [14] EBJP3MNA Build against nix-master
  • [15] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
  • [16] YSZQ3ORR Fix build
  • [17] HH3LID6L Re-implement log size limits
  • [18] 6LA56M37 Fix build against current Nix master
  • [19] AUMIJSEO Fix remote building on Nix 1.10
  • [20] 2GRQJZT6 hydra-queue-runner: Support running in a NixOS container
  • [21] 5AIYUMTB Basic remote building

Change contents

  • edit in src/hydra-queue-runner/build-remote.cc at line 213
    [4.328][4.328:364]()
    bool sendDerivation = true;
  • edit in src/hydra-queue-runner/build-remote.cc at line 225
    [4.776][4.39:352](),[4.917][4.39:352](),[4.352][4.973:1013](),[4.973][4.973:1013]()
    // Always send the derivation to localhost, since it's a
    // no-op anyway but we might not be privileged to use
    // cmdBuildDerivation (e.g. if we're running in a NixOS
    // container).
    if (GET_PROTOCOL_MINOR(remoteVersion) >= 1 && !machine->isLocalhost())
    sendDerivation = false;
  • replacement in src/hydra-queue-runner/build-remote.cc at line 249
    [4.651][4.1703:1731](),[4.1731][4.860:902](),[4.891][4.1773:1786](),[4.902][4.1773:1786](),[4.1398][4.1773:1786](),[4.1773][4.1773:1786](),[4.1786][4.892:942](),[4.942][4.903:937]()
    if (sendDerivation)
    inputs.insert(step->drvPath);
    else
    for (auto & p : step->drv->inputSrcs)
    inputs.insert(p);
    [4.651]
    [4.797]
    for (auto & p : step->drv->inputSrcs)
    inputs.insert(p);
  • replacement in src/hydra-queue-runner/build-remote.cc at line 306
    [4.355][4.2174:2237](),[4.2237][3.0:79](),[3.79][4.2309:2475](),[4.232][4.2309:2475](),[4.1108][4.2309:2475](),[4.2309][4.2309:2475]()
    if (sendDerivation) {
    to << cmdBuildPaths;
    worker_proto::write(*localStore, to, StorePathSet{step->drvPath});
    } else {
    to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
    writeDerivation(to, *localStore, basicDrv);
    }
    [4.355]
    [4.3615]
    to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
    writeDerivation(to, *localStore, basicDrv);
  • replacement in src/hydra-queue-runner/build-remote.cc at line 325
    [4.1154][4.3975:4028](),[4.4028][4.1109:1202](),[4.1202][4.4136:4551](),[4.4136][4.4136:4551](),[4.4551][4.116:130](),[4.116][4.116:130](),[4.130][4.4552:4660](),[4.4660][4.259:427](),[4.427][2.0:360](),[2.360][4.427:441](),[4.427][4.427:441](),[4.441][4.4660:5695](),[4.4660][4.4660:5695](),[4.5945][4.5945:6391](),[4.6391][4.428:656](),[4.656][4.6391:6521](),[4.6391][4.6391:6521]()
    if (sendDerivation) {
    if (res) {
    result.errorMsg = fmt("%s on ‘%s’", readString(from), machine->sshName);
    if (res == 100) {
    result.stepStatus = bsFailed;
    result.canCache = true;
    }
    else if (res == 101) {
    result.stepStatus = bsTimedOut;
    }
    else {
    result.stepStatus = bsAborted;
    result.canRetry = true;
    }
    return;
    }
    result.stepStatus = bsSuccess;
    } else {
    result.errorMsg = readString(from);
    if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
    result.timesBuilt = readInt(from);
    result.isNonDeterministic = readInt(from);
    auto start = readInt(from);
    auto stop = readInt(from);
    if (start && start) {
    /* Note: this represents the duration of a single
    round, rather than all rounds. */
    result.startTime = start;
    result.stopTime = stop;
    }
    }
    switch ((BuildResult::Status) res) {
    case BuildResult::Built:
    result.stepStatus = bsSuccess;
    break;
    case BuildResult::Substituted:
    case BuildResult::AlreadyValid:
    result.stepStatus = bsSuccess;
    result.isCached = true;
    break;
    case BuildResult::PermanentFailure:
    result.stepStatus = bsFailed;
    result.canCache = true;
    result.errorMsg = "";
    break;
    case BuildResult::InputRejected:
    case BuildResult::OutputRejected:
    result.stepStatus = bsFailed;
    result.canCache = true;
    break;
    case BuildResult::TransientFailure:
    result.stepStatus = bsFailed;
    result.canRetry = true;
    result.errorMsg = "";
    break;
    case BuildResult::TimedOut:
    result.stepStatus = bsTimedOut;
    result.errorMsg = "";
    break;
    case BuildResult::MiscFailure:
    result.stepStatus = bsAborted;
    result.canRetry = true;
    break;
    case BuildResult::LogLimitExceeded:
    result.stepStatus = bsLogLimitExceeded;
    break;
    case BuildResult::NotDeterministic:
    result.stepStatus = bsNotDeterministic;
    result.canRetry = false;
    result.canCache = true;
    break;
    default:
    result.stepStatus = bsAborted;
    break;
    [4.1154]
    [4.333]
    result.errorMsg = readString(from);
    if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
    result.timesBuilt = readInt(from);
    result.isNonDeterministic = readInt(from);
    auto start = readInt(from);
    auto stop = readInt(from);
    if (start && start) {
    /* Note: this represents the duration of a single
    round, rather than all rounds. */
    result.startTime = start;
    result.stopTime = stop;
  • replacement in src/hydra-queue-runner/build-remote.cc at line 337
    [4.347][4.6522:6578]()
    if (result.stepStatus != bsSuccess) return;
    [4.347]
    [4.1529]
    }
    switch ((BuildResult::Status) res) {
    case BuildResult::Built:
    result.stepStatus = bsSuccess;
    break;
    case BuildResult::Substituted:
    case BuildResult::AlreadyValid:
    result.stepStatus = bsSuccess;
    result.isCached = true;
    break;
    case BuildResult::PermanentFailure:
    result.stepStatus = bsFailed;
    result.canCache = true;
    result.errorMsg = "";
    break;
    case BuildResult::InputRejected:
    case BuildResult::OutputRejected:
    result.stepStatus = bsFailed;
    result.canCache = true;
    break;
    case BuildResult::TransientFailure:
    result.stepStatus = bsFailed;
    result.canRetry = true;
    result.errorMsg = "";
    break;
    case BuildResult::TimedOut:
    result.stepStatus = bsTimedOut;
    result.errorMsg = "";
    break;
    case BuildResult::MiscFailure:
    result.stepStatus = bsAborted;
    result.canRetry = true;
    break;
    case BuildResult::LogLimitExceeded:
    result.stepStatus = bsLogLimitExceeded;
    break;
    case BuildResult::NotDeterministic:
    result.stepStatus = bsNotDeterministic;
    result.canRetry = false;
    result.canCache = true;
    break;
    default:
    result.stepStatus = bsAborted;
    break;
  • edit in src/hydra-queue-runner/build-remote.cc at line 382
    [4.1539]
    [4.6579]
    if (result.stepStatus != bsSuccess) return;