pijul nest
guest › [sign in] [explore]

Update to reflect BinaryCacheStore changes

[?]
Oct 7, 2016, 6:23 PM
IKJBYIGY3JABFXD2KOAQNA52ULE6VR4PBFKHWAHHBITXX5KC76MAC

Dependencies

  • [2] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
  • [3] YTAYNN7V Queue monitor: Bail out earlier if a step has failed previously
  • [4] HB7JA5WV hydra-module: fix useSubstitutes configuration option
  • [5] PDI5YPBY Use store-api for binary cache instantiation
  • [6] OG3Z3QGC Namespace cleanup
  • [7] TTBLPQAJ Keep track of wait time per system type
  • [8] V2UCCYN3 hydra-queue-runner: Get store mode configuration from hydra.conf
  • [9] EYR3EW6J Keep stats for the Hydra auto scaler
  • [10] H7SZRHUB Use nix::willBuildLocally()
  • [11] 7XOH2DGH hydra-module.nix: Give types to options
  • [12] A4A5KO5Q Add error/warnings for deprecated store specification
  • [13] RND7XFNH getQueuedBuilds(): Periodically stop to handle priority bumps
  • [14] GJV2J5HX Pool local store connections
  • [15] HJOEIMLR Refactor
  • [16] DKJFD6JN Process Nix API changes
  • [17] 36YF4WOZ Remove unnecessary call to hydra-queue-runner --unlock
  • [18] 2ANV36PB Hackery
  • [19] B2L4T3X6 Sync with Nix
  • [20] TTZ26BJQ Unify Hydra's NixOS module with the one used for hydra.nixos.org
  • [21] BG6PEOB2 Make the output size limit configurable
  • [22] SODOV2CM Automatically reload $NIX_REMOTE_SYSTEMS when it changes
  • [23] EOO4EFWD Use a single BinaryCacheStore for all threads
  • [24] BRAESISH Warn if PostgreSQL appears stalled
  • [25] K5G5GZY7 Guard against concurrent invocations of hydra-queue-runner
  • [26] 5AIYUMTB Basic remote building
  • [27] WE5Q2NVI Allow build to be bumped to the front of the queue via the web interface
  • [28] ACBS7C6Q hydra-queue-runner: Detect changes to the scheduling shares
  • [29] SOB276BA Keep some statistics for the binary cache stores
  • [30] G7LZJ4VA hydra-module.nix: add 'useSubstitutes' option to control use of binary caches
  • [31] 24BMQDZA Start of single-process hydra-queue-runner
  • [32] VQISTKOP hydra-queue-runner: Use substitutes
  • [33] 4HDPYRJ6 Update the hydra-queue-runner unit
  • [34] ZYEFL36Q Ported/merged properties from the TUD configuration's hydra-module
  • [35] MHVIT4JY Split hydra-queue-runner.cc more
  • [36] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
  • [*] D3MDJONY
  • [*] KKHVM4KA Move GC roots to /nix/var/nix/gcroots/hydra
  • [*] 4J33F22T hydra-module.nix: Make it possible to add extra config lines

Change contents

  • edit in hydra-module.nix at line 233
    [39.232]
    [40.373]
    use-substitutes ${if cfg.useSubstitutes then "1" else "0"}
  • replacement in hydra-module.nix at line 329
    [6.1684]→[4.0:169](∅→∅)
    { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes ${if cfg.useSubstitutes then "true" else "false"}";
    [6.1684]
    [6.85]
    { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v";
  • replacement in src/hydra-queue-runner/build-remote.cc at line 205
    [2.2491]→[2.2491:2542](∅→∅)
    destStore->buildPaths(basicDrv.inputSrcs);
    [2.2491]
    [6.3876]
    copyClosure(ref<Store>(localStore), destStore, basicDrv.inputSrcs);
  • edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 71
    [6.566]→[6.1542:1546](∅→∅),[6.979]→[6.1542:1546](∅→∅),[6.7892]→[6.1542:1546](∅→∅),[6.1546]→[6.755:789](∅→∅),[6.659]→[6.1580:1582](∅→∅),[6.789]→[6.1580:1582](∅→∅),[6.1580]→[6.1580:1582](∅→∅),[6.1582]→[6.790:826](∅→∅)
    }
    ref<Store> State::getLocalStore()
    {
    return ref<Store>(_localStore);
  • replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 794
    [6.828]→[6.828:859](∅→∅)
    _localStore = openStore();
    [6.828]
    [6.4619]
    localStore = openStore();
  • replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 797
    [5.67]→[6.860:894](∅→∅),[6.662]→[6.860:894](∅→∅)
    _destStore = _localStore;
    [5.67]
    [5.68]
    _destStore = localStore;
  • edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 801
    [6.1249]
    [6.1249]
    auto isTrue = [](const std::string & s) {
    return s == "1" || s == "true";
    };
    useSubstitutes = isTrue(hydraConfig["use-substitutes"]);
  • edit in src/hydra-queue-runner/queue-monitor.cc at line 32
    [6.20720]→[6.12280:12319](∅→∅)
    auto localStore = getLocalStore();
  • replacement in src/hydra-queue-runner/queue-monitor.cc at line 37
    [6.20820]→[6.1270:1350](∅→∅)
    bool done = getQueuedBuilds(*conn, localStore, destStore, lastBuildId);
    [6.20820]
    [6.20872]
    bool done = getQueuedBuilds(*conn, destStore, lastBuildId);
  • replacement in src/hydra-queue-runner/queue-monitor.cc at line 71
    [3.127]→[6.12438:12508](∅→∅),[6.21512]→[6.12438:12508](∅→∅)
    bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
    [3.127]
    [6.12508]
    bool State::getQueuedBuilds(Connection & conn,
  • edit in src/hydra-queue-runner/queue-monitor.cc at line 417
    [6.1605]→[6.1605:1631](∅→∅)
    PathSet missingPaths;
  • replacement in src/hydra-queue-runner/queue-monitor.cc at line 421
    [6.1712]→[6.1712:1760](∅→∅)
    missingPaths.insert(i.second.path);
    [6.1712]
    [6.1760]
    }
    /* Try to copy the missing paths from the local store or from
    substitutes. */
    if (!missing.empty()) {
    size_t avail = 0;
    for (auto & i : missing) {
    if (/* localStore != destStore && */ localStore->isValidPath(i.second.path))
    avail++;
    else if (useSubstitutes) {
    SubstitutablePathInfos infos;
    localStore->querySubstitutablePathInfos({i.second.path}, infos);
    if (infos.size() == 1)
    avail++;
    }
  • replacement in src/hydra-queue-runner/queue-monitor.cc at line 439
    [6.1771]→[6.1771:2120](∅→∅),[6.2120]→[6.12926:13004](∅→∅),[6.13004]→[6.2185:2236](∅→∅),[6.2185]→[6.2185:2236](∅→∅)
    /* Try to substitute the missing paths. Note: can't use the more
    efficient querySubstitutablePaths() here because upstream Hydra
    servers don't allow it (they have "WantMassQuery: 0"). */
    assert(missing.size() == missingPaths.size());
    if (!missing.empty() && settings.useSubstitutes) {
    SubstitutablePathInfos infos;
    destStore->querySubstitutablePathInfos(missingPaths, infos); // FIXME
    if (infos.size() == missingPaths.size()) {
    [6.1771]
    [6.2236]
    if (missing.size() == avail) {
  • replacement in src/hydra-queue-runner/queue-monitor.cc at line 443
    [6.2323]→[6.2323:2442](∅→∅)
    printMsg(lvlInfo, format("substituting output ‘%1%’ of ‘%2%’") % i.second.path % drvPath);
    [6.2323]
    [6.2442]
    time_t startTime = time(0);
    if (localStore->isValidPath(i.second.path))
    printInfo("copying output ‘%1%’ of ‘%2%’ from local store", i.second.path, drvPath);
    else {
    printInfo("substituting output ‘%1%’ of ‘%2%’", i.second.path, drvPath);
    localStore->ensurePath(i.second.path);
    // FIXME: should copy directly from substituter to destStore.
    }
    copyClosure(ref<Store>(localStore), destStore, {i.second.path});
  • edit in src/hydra-queue-runner/queue-monitor.cc at line 455
    [6.2443]→[6.2443:2491](∅→∅),[6.2491]→[6.13005:13063](∅→∅)
    time_t startTime = time(0);
    destStore->ensurePath(i.second.path);
  • edit in src/hydra-queue-runner/queue-monitor.cc at line 465
    [6.2872]
    [6.2872]
    printError("while copying/substituting output ‘%s’ of ‘%s’: %s", i.second.path, drvPath, e.what());
  • edit in src/hydra-queue-runner/state.hh at line 259
    [6.1870]
    [6.4416]
    bool useSubstitutes = false;
  • replacement in src/hydra-queue-runner/state.hh at line 379
    [6.6778]→[6.2286:2331](∅→∅)
    std::shared_ptr<nix::Store> _localStore;
    [6.6778]
    [6.1458]
    std::shared_ptr<nix::Store> localStore;
  • edit in src/hydra-queue-runner/state.hh at line 405
    [6.946]→[6.946:947](∅→∅),[6.947]→[6.13176:13272](∅→∅),[6.13176]→[6.13176:13272](∅→∅),[6.13272]→[6.2332:2374](∅→∅)
    /* Return a store object that can access derivations produced by
    hydra-evaluator. */
    nix::ref<nix::Store> getLocalStore();
  • replacement in src/hydra-queue-runner/state.hh at line 437
    [6.2260]→[6.13413:13490](∅→∅)
    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> localStore,
    [6.2260]
    [6.13490]
    bool getQueuedBuilds(Connection & conn,