Support hydra-build-products on binary cache stores

Dependencies

• [2] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [3] NJJ7H64S Very basic multi-threaded queue runner
• [4] 24BMQDZA Start of single-process hydra-queue-runner
• [5] GS4BE6TB Asynchronously compress build logs
• [6] YHP5DSOO Improve parsing of hydra-build-products
• [7] T5BIOVJE Add support for tracking custom metrics
• [8] WHULPA6S Handle failure with output
• [9] ZK76B5ZZ Load the queue in order of global priority
• [10] DKJFD6JN Process Nix API changes
• [11] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
• [12] MHVIT4JY Split hydra-queue-runner.cc more

Change contents

• edit in src/hydra-queue-runner/build-result.cc at line 5
  [3.20]

  [3.531]
  + #include "fs-accessor.hh"
• edit in src/hydra-queue-runner/build-result.cc at line 8
  [3.553]→[3.0:402](∅→∅)
  −

  −

  − static std::tuple<bool, string> secureRead(Path fileName)

  − {

  − auto fail = std::make_tuple(false, "");

  −

  − if (!pathExists(fileName)) return fail;

  −

  − try {

  − /* For security, resolve symlinks. */

  − fileName = canonPath(fileName, true);

  − if (!isInStore(fileName)) return fail;

  − return std::make_tuple(true, readFile(fileName));

  − } catch (Error & e) { return fail; }

  − }
• replacement in src/hydra-queue-runner/build-result.cc at line 10
  [3.555]→[3.419:493](∅→∅)
  − BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
  [3.555]

  [3.606]
  + BuildOutput getBuildOutput(nix::ref<Store> store,

  + nix::ref<nix::FSAccessor> accessor, const Derivation & drv)
• edit in src/hydra-queue-runner/build-result.cc at line 31
  [3.1146]→[2.1107:1113](∅→∅)
  − #if 0
• replacement in src/hydra-queue-runner/build-result.cc at line 42
  [3.58]→[3.58:113](∅→∅)
  − if (pathExists(failedFile)) res.failed = true;
  [3.58]

  [3.113]
  + if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)

  + res.failed = true;
• replacement in src/hydra-queue-runner/build-result.cc at line 45
  [3.114]→[3.403:523](∅→∅)
  − auto file = secureRead(output + "/nix-support/hydra-build-products");

  − if (!std::get<0>(file)) continue;
  [3.114]

  [3.523]
  + Path productsFile = output + "/nix-support/hydra-build-products";

  + if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)

  + continue;
• replacement in src/hydra-queue-runner/build-result.cc at line 51
  [3.1339]→[3.525:604](∅→∅)
  − for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
  [3.1339]

  [3.640]
  + for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {
• edit in src/hydra-queue-runner/build-result.cc at line 64
  [3.629]

  [3.2316]
  + // FIXME: should we disallow products referring to other

  + // store paths, or that are outside the input closure?
• replacement in src/hydra-queue-runner/build-result.cc at line 67
  [3.2388]→[3.976:1102](∅→∅),[3.1102]→[3.2446:2527](∅→∅),[3.2446]→[3.2446:2527](∅→∅)
  − try {

  − product.path = canonPath(product.path, true);

  − } catch (Error & e) { continue; }

  − if (!isInStore(product.path) || !pathExists(product.path)) continue;
  [3.2388]

  [3.2527]
  + product.path = canonPath(product.path);

  + if (!isInStore(product.path)) continue;
• replacement in src/hydra-queue-runner/build-result.cc at line 70
  [3.2528]→[3.2528:2628](∅→∅)
  − /* FIXME: check that the path is in the input closure

  − of the build? */
  [3.2528]

  [3.2628]
  + auto st = accessor->stat(product.path);

  + if (st.type == FSAccessor::Type::tMissing) continue;
• edit in src/hydra-queue-runner/build-result.cc at line 74
  [3.2712]→[3.2712:2876](∅→∅)
  −

  − struct stat st;

  − if (stat(product.path.c_str(), &st))

  − throw SysError(format("getting status of ‘%1%’") % product.path);
• replacement in src/hydra-queue-runner/build-result.cc at line 75
  [3.2877]→[3.2877:2916](∅→∅)
  − if (S_ISREG(st.st_mode)) {
  [3.2877]

  [3.2916]
  + if (st.type == FSAccessor::Type::tRegular) {
• replacement in src/hydra-queue-runner/build-result.cc at line 77
  [3.2958]→[3.2958:3143](∅→∅)
  − product.fileSize = st.st_size;

  − product.sha1hash = hashFile(htSHA1, product.path);

  − product.sha256hash = hashFile(htSHA256, product.path);
  [3.2958]

  [3.3143]
  + product.fileSize = st.fileSize;

  + auto contents = accessor->readFile(product.path);

  + product.sha1hash = hashString(htSHA1, contents);

  + product.sha256hash = hashString(htSHA256, contents);
• edit in src/hydra-queue-runner/build-result.cc at line 86
  [3.3219]→[2.1114:1121](∅→∅)
  − #endif
• replacement in src/hydra-queue-runner/build-result.cc at line 97
  [3.3665]→[2.1122:1128](∅→∅),[2.1128]→[3.3665:3865](∅→∅),[3.3665]→[3.3665:3865](∅→∅),[3.3865]→[2.1129:1136](∅→∅)
  − #if 0

  − struct stat st;

  − if (stat(product.path.c_str(), &st))

  − throw SysError(format("getting status of ‘%1%’") % product.path);

  − if (S_ISDIR(st.st_mode))

  − #endif
  [3.3665]

  [3.3865]
  + auto st = accessor->stat(product.path);

  + if (st.type == FSAccessor::Type::tMissing)

  + throw Error(format("getting status of ‘%1%’") % product.path);

  + if (st.type == FSAccessor::Type::tDirectory)
• edit in src/hydra-queue-runner/build-result.cc at line 105
  [3.3931]→[2.1137:1143](∅→∅)
  − #if 0
• replacement in src/hydra-queue-runner/build-result.cc at line 108
  [3.4104]→[3.4104:4142](∅→∅)
  − if (!pathExists(p)) continue;
  [3.4104]

  [3.1103]
  + if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
• replacement in src/hydra-queue-runner/build-result.cc at line 110
  [3.1117]→[3.1117:1166](∅→∅)
  − res.releaseName = trim(readFile(p));
  [3.1117]

  [3.1166]
  + res.releaseName = trim(accessor->readFile(p));
• replacement in src/hydra-queue-runner/build-result.cc at line 117
  [3.664]→[3.664:814](∅→∅)
  − auto file = secureRead(output + "/nix-support/hydra-metrics");

  − for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
  [3.664]

  [3.814]
  + Path metricsFile = output + "/nix-support/hydra-metrics";

  + if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;

  + for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
• edit in src/hydra-queue-runner/build-result.cc at line 129
  [3.1208]→[2.1144:1151](∅→∅)
  − #endif
• edit in src/hydra-queue-runner/build-result.hh at line 7
  [3.4389]

  [3.4389]
  + #include "store-api.hh"
• replacement in src/hydra-queue-runner/build-result.hh at line 41
  [3.4741]→[3.545:630](∅→∅)
  − BuildOutput getBuildOutput(nix::ref<nix::Store> store, const nix::Derivation & drv);
  [3.4741]
  + BuildOutput getBuildOutput(nix::ref<nix::Store> store,

  + nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);
• replacement in src/hydra-queue-runner/builder.cc at line 129
  [3.4608]→[2.1433:1507](∅→∅)
  − if (result.success()) res = getBuildOutput(destStore, step->drv);
  [3.4608]

  [3.4678]
  + if (result.success()) res = getBuildOutput(destStore, destStore->getFSAccessor(), step->drv);
• replacement in src/hydra-queue-runner/queue-monitor.cc at line 162
  [3.24924]→[2.12757:12819](∅→∅)
  − BuildOutput res = getBuildOutput(destStore, drv);
  [3.24924]

  [3.24982]
  + BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv);