pijul nest
guest › [sign in] [explore]

hydra-queue-runner: don't try to distribute builds on localhost

[?]
May 3, 2020, 4:05 AM
H46LUUZY3N7DDOIZFDZ7SJP76BICH4G6Z4SC7GBPMIKHW2JPZRTAC

Dependencies

  • [2] WV4SSAIY Build against nix-master
  • [3] EBJP3MNA Build against nix-master
  • [4] 2GRQJZT6 hydra-queue-runner: Support running in a NixOS container
  • [5] BG6PEOB2 Make the output size limit configurable
  • [6] FJCKDJKB Acquire the send lock only while actually sending
  • [7] GS4BE6TB Asynchronously compress build logs
  • [8] A2GL5FOZ Moar stats
  • [9] 2BPAQMDH hydra-queue-runner: better error message if nix-store can not be started
  • [10] 7VQ4ALFY Update "make check" for the new queue runner
  • [11] CNLNT3T4 Allow only 1 thread to send a closure to a given machine at the same time
  • [12] 6TY4LNHH Finish copyClosure
  • [13] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
  • [14] 4VYY2ADP Allow the machines file to specify host public keys
  • [15] UVNTWTWG Prevent download of NARs we just uploaded
  • [16] 5AIYUMTB Basic remote building
  • [17] FITVNQ2S Keep track of the time we spend copying to/from build machines
  • [18] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
  • [19] OJFOBJKF Add FIXME
  • [20] XCDTFZUY hydra-queue-runner: Fix build
  • [21] OKEMZJLS Bump Nix
  • [22] LSUX6IQR Update to latest nixUnstable
  • [23] LE4VZIY5 More stats
  • [24] SL3WSRAC hydra-queue-runner: Limit memory usage
  • [25] 4NU5JJFD Truncate the log just before starting the remote build
  • [26] G7KWXSFM Distinguish build step states
  • [27] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
  • [28] MB3TISH2 Rate-limit the number of threads copying closures at the same time
  • [29] 6EO3HVNA Merge remote-tracking branch 'origin/master' into binary-cache
  • [30] DKJFD6JN Process Nix API changes
  • [31] N4IROACV Move buildRemote() into State

Change contents

  • replacement in src/hydra-queue-runner/build-remote.cc at line 53
    [4.103]→[4.333:389](∅→∅),[4.162]→[4.333:389](∅→∅),[4.333]→[4.333:389](∅→∅)
    argv = {"nix-store", "--serve", "--write"};
    [4.103]
    [4.104]
    argv = {"nix-store", "--builders", "", "--serve", "--write"};
  • edit in src/hydra-queue-runner/build-remote.cc at line 85
    [4.78]→[2.320:382](∅→∅)
    FdSource & from, FdSink & to, const StorePathSet & paths,
  • edit in src/hydra-queue-runner/build-remote.cc at line 89
    [4.1441]→[3.890:916](∅→∅)
    StorePathSet closure;
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 89
    [2.409]
    [4.1462]
  • edit in src/hydra-queue-runner/build-remote.cc at line 99
    [4.55]→[3.917:1016](∅→∅)
    to << cmdQueryValidPaths << 1 << useSubstitutes;
    writeStorePaths(*destStore, to, closure);
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 99
    [2.509]
    [4.1984]
  • edit in src/hydra-queue-runner/build-remote.cc at line 103
    [4.2087]→[2.510:577](∅→∅)
    auto present = readStorePaths<StorePathSet>(*destStore, from);
  • edit in src/hydra-queue-runner/build-remote.cc at line 107
    [4.83]→[2.578:631](∅→∅)
    auto sorted = destStore->topoSortPaths(closure);
  • edit in src/hydra-queue-runner/build-remote.cc at line 109
    [4.135]→[2.632:658](∅→∅)
    StorePathSet missing;
  • edit in src/hydra-queue-runner/build-remote.cc at line 112
    [4.214]→[3.1166:1226](∅→∅)
    if (!present.count(*i)) missing.insert(i->clone());
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 112
    [2.719]
    [4.398]
  • edit in src/hydra-queue-runner/build-remote.cc at line 136
    [4.177]→[3.1227:1271](∅→∅)
    string base(step->drvPath.to_string());
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 136
    [2.764]
    [4.0]
  • edit in src/hydra-queue-runner/build-remote.cc at line 221
    [4.321]→[2.765:840](∅→∅)
    StorePathSet inputs;
    BasicDerivation basicDrv(*step->drv);
  • edit in src/hydra-queue-runner/build-remote.cc at line 225
    [4.1731]→[2.841:891](∅→∅)
    inputs.insert(step->drvPath.clone());
  • edit in src/hydra-queue-runner/build-remote.cc at line 229
    [4.1786]→[3.1399:1491](∅→∅)
    for (auto & p : step->drv->inputSrcs)
    inputs.insert(p.clone());
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 229
    [2.984]
    [4.797]
  • edit in src/hydra-queue-runner/build-remote.cc at line 230
    [4.798]→[2.985:1137](∅→∅)
    for (auto & input : step->drv->inputDrvs) {
    Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
  • edit in src/hydra-queue-runner/build-remote.cc at line 237
    [4.2132]→[3.1645:1767](∅→∅)
    inputs.insert(i->second.path.clone());
    basicDrv.inputSrcs.insert(i->second.path.clone());
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 237
    [2.1260]
    [4.2238]
  • edit in src/hydra-queue-runner/build-remote.cc at line 246
    [4.414]→[3.1768:1873](∅→∅)
    copyClosure(ref<Store>(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs);
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 246
    [2.1366]
    [4.3876]
  • edit in src/hydra-queue-runner/build-remote.cc at line 252
    [4.140]→[2.1367:1519](∅→∅)
    printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
    localStore->printStorePath(step->drvPath), machine->sshName);
  • edit in src/hydra-queue-runner/build-remote.cc at line 277
    [4.3339]→[2.1520:1666](∅→∅)
    printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
    localStore->printStorePath(step->drvPath),
    machine->sshName);
  • edit in src/hydra-queue-runner/build-remote.cc at line 283
    [4.355]→[2.1667:1968](∅→∅)
    if (sendDerivation) {
    to << cmdBuildPaths;
    writeStorePaths(*localStore, to, singleton(step->drvPath));
    } else {
    to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
    writeDerivation(to, *localStore, basicDrv);
    }
  • edit in src/hydra-queue-runner/build-remote.cc at line 390
    [4.6745]→[2.1969:2141](∅→∅)
    printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
    localStore->printStorePath(step->drvPath), machine->sshName);
  • edit in src/hydra-queue-runner/build-remote.cc at line 406
    [4.2001]→[2.2142:2195](∅→∅)
    auto outputs = step->drv->outputPaths();
  • edit in src/hydra-queue-runner/build-remote.cc at line 412
    [4.7393]→[3.2703:2795](∅→∅)
    to << cmdQueryPathInfos;
    writeStorePaths(*localStore, to, outputs);
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 412
    [2.2288]
    [4.7441]
  • edit in src/hydra-queue-runner/build-remote.cc at line 426
    [4.104]→[2.2289:2468](∅→∅)
    printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
    localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);
  • edit in src/hydra-queue-runner/build-remote.cc at line 441
    [4.8575]→[2.2469:2647](∅→∅)
    printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s",
    resMs, totalNarSize, localStore->printStorePath(step->drvPath));
  • edit in src/hydra-queue-runner/build-remote.cc at line 446
    [4.218]→[3.3155:3249](∅→∅)
    to << cmdExportPaths << 0;
    writeStorePaths(*localStore, to, outputs);
  • resolve order conflict in src/hydra-queue-runner/build-remote.cc at line 446
    [2.2742]
    [4.8846]