hydra-queue-runner: Support generating a signed binary cache

Dependencies

• [2] 3FQ65IXO hydra-queue-runner: Compress binary cache NARs using xz
• [3] 24BMQDZA Start of single-process hydra-queue-runner
• [4] 5AIYUMTB Basic remote building
• [5] 73YR46NJ hydra-queue-runner: Write directly to a binary cache

Change contents

• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 36
  [3.1660]→[3.1660:1737](∅→∅)
  − return make_ref<LocalBinaryCache>(getLocalStore(), "/tmp/binary-cache");
  [3.1660]

  [3.7503]
  + return make_ref<LocalBinaryCache>(getLocalStore(),

  + "/tmp/binary-cache",

  + "/home/eelco/Misc/Keys/test.nixos.org/secret",

  + "/home/eelco/Misc/Keys/test.nixos.org/public");
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 7
  [3.1984]

  [3.1984]
  + #include "nar-info.hh"
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 12
  [3.2032]→[3.2032:2180](∅→∅)
  − LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir)

  − : localStore(localStore), binaryCacheDir(binaryCacheDir)
  [3.2032]

  [3.2180]
  + LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,

  + const Path & secretKeyFile, const Path & publicKeyFile)

  + : localStore(localStore)

  + , binaryCacheDir(binaryCacheDir)
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 18
  [3.2223]

  [3.2223]
  +

  + Path cacheInfoFile = binaryCacheDir + "/nix-cache-info";

  + if (!pathExists(cacheInfoFile))

  + writeFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");

  +

  + if (secretKeyFile != "")

  + secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));

  +

  + if (publicKeyFile != "") {

  + publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);

  + auto key = PublicKey(readFile(publicKeyFile));

  + publicKeys->emplace(key.name, key);

  + }
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 54
  [2.118]

  [2.118]
  +

  + NarInfo narInfo(info);
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 57
  [2.119]→[3.2806:2885](∅→∅),[3.2806]→[3.2806:2885](∅→∅)
  − size_t narSize = nar.size();

  − Hash narHash = hashString(htSHA256, nar);
  [2.119]

  [3.2885]
  + narInfo.narSize = nar.size();

  + narInfo.narHash = hashString(htSHA256, nar);
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 60
  [3.2886]→[3.2886:2947](∅→∅)
  − if (info.hash.type != htUnknown && info.hash != narHash)
  [3.2886]

  [3.2947]
  + if (info.narHash.type != htUnknown && info.narHash != narInfo.narHash)
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 64
  [3.3138]→[3.3138:3170](∅→∅)
  − % info.path % narSize);
  [3.3138]

  [2.120]
  + % info.path % info.narSize);
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 67
  [2.149]

  [2.149]
  + narInfo.compression = "xz";
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 69
  [2.185]→[2.185:235](∅→∅)
  − Hash narXzHash = hashString(htSHA256, narXz);
  [2.185]

  [3.3170]
  + narInfo.fileHash = hashString(htSHA256, narXz);

  + narInfo.fileSize = narXz.size();
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 73
  [3.3212]→[2.236:305](∅→∅),[2.305]→[3.3274:3328](∅→∅),[3.3274]→[3.3274:3328](∅→∅)
  − string narFileRel = "nar/" + printHash32(narXzHash) + ".nar.xz";

  − Path narFile = binaryCacheDir + "/" + narFileRel;
  [3.3212]

  [2.306]
  + narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";

  + Path narFile = binaryCacheDir + "/" + narInfo.url;
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 78
  [3.3431]→[2.366:460](∅→∅),[2.460]→[3.3481:3482](∅→∅),[3.3481]→[3.3481:3482](∅→∅),[3.3482]→[2.461:944](∅→∅)
  − Strings refs;

  − for (auto & r : info.references)

  − refs.push_back(baseNameOf(r));

  −

  − std::string narInfo;

  − narInfo += "StorePath: " + info.path + "\n";

  − narInfo += "URL: " + narFileRel + "\n";

  − narInfo += "Compression: xz\n";

  − narInfo += "FileHash: sha256:" + printHash32(narXzHash) + "\n";

  − narInfo += "FileSize: " + std::to_string(narXz.size()) + "\n";

  − narInfo += "NarHash: sha256:" + printHash32(narHash) + "\n";

  − narInfo += "NarSize: " + std::to_string(narSize) + "\n";

  − narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
  [3.3431]

  [3.3518]
  + if (secretKey) narInfo.sign(*secretKey);
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 80
  [3.3519]→[2.945:973](∅→∅),[2.973]→[3.4136:4137](∅→∅),[3.4136]→[3.4136:4137](∅→∅),[3.4137]→[2.974:1013](∅→∅)
  − // FIXME: add signature

  −

  − atomicWrite(narInfoFile, narInfo);
  [3.3519]

  [3.4219]
  + atomicWrite(narInfoFile, narInfo.to_string());
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 83
  [3.4222]→[3.4222:4302](∅→∅)
  − LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
  [3.4222]

  [3.4302]
  + NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 85
  [3.4304]→[3.4304:4322](∅→∅)
  − NarInfo res;

  −
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 86
  [3.4372]→[3.4372:5476](∅→∅),[3.5476]→[2.1014:1103](∅→∅)
  − if (!pathExists(narInfoFile))

  − abort();

  − std::string narInfo = readFile(narInfoFile);

  −

  − auto corrupt = [&]() {

  − throw Error(format("corrupt NAR info file ‘%1%’") % narInfoFile);

  − };

  −

  − size_t pos = 0;

  − while (pos < narInfo.size()) {

  −

  − size_t colon = narInfo.find(':', pos);

  − if (colon == std::string::npos) corrupt();

  −

  − std::string name(narInfo, pos, colon - pos);

  −

  − size_t eol = narInfo.find('\n', colon + 2);

  − if (eol == std::string::npos) corrupt();

  −

  − std::string value(narInfo, colon + 2, eol - colon - 2);

  −

  − if (name == "StorePath") {

  − res.info.path = value;

  − if (value != storePath) corrupt();

  − res.info.path = value;

  − }

  − else if (name == "References") {

  − auto refs = tokenizeString<Strings>(value, " ");

  − if (!res.info.references.empty()) corrupt();

  − for (auto & r : refs)

  − res.info.references.insert(settings.nixStore + "/" + r);

  − }

  − else if (name == "URL") {

  − res.narUrl = value;

  − }

  − else if (name == "Compression") {

  − res.compression = value;

  − }
  [3.4372]

  [3.5476]
  + NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile);

  + assert(narInfo.path == storePath);
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 89
  [3.5477]→[3.5477:5500](∅→∅)
  − pos = eol + 1;
  [3.5477]

  [3.5500]
  + if (publicKeys) {

  + if (!narInfo.checkSignature(*publicKeys))

  + throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
• edit in src/hydra-queue-runner/local-binary-cache.cc at line 93
  [3.5506]→[3.5506:5571](∅→∅)
  −

  − if (res.info.path.empty() || res.narUrl.empty()) corrupt();
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 94
  [3.5572]→[3.5572:5588](∅→∅)
  − return res;
  [3.5572]

  [3.5588]
  + return narInfo;
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 99
  [3.5652]→[3.5652:5821](∅→∅)
  − Path narInfoFile = narInfoFileFor(storePath);

  −

  − printMsg(lvlDebug, format("checking %1% -> %2%") % storePath % narInfoFile);

  −

  − return pathExists(narInfoFile);
  [3.5652]

  [3.5821]
  + return pathExists(narInfoFileFor(storePath));
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 108
  [3.5968]→[3.5968:6028](∅→∅)
  − auto nar = readFile(binaryCacheDir + "/" + res.narUrl);
  [3.5968]

  [2.1104]
  + auto nar = readFile(binaryCacheDir + "/" + res.url);
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 127
  [3.6258]→[3.6258:6344](∅→∅)
  − sink << exportMagic << storePath << res.info.references << res.info.deriver << 0;
  [3.6258]

  [3.6344]
  + sink << exportMagic << storePath << res.references << res.deriver << 0;
• replacement in src/hydra-queue-runner/local-binary-cache.cc at line 192
  [3.7842]→[3.7842:7882](∅→∅)
  − return readNarInfo(storePath).info;
  [3.7842]

  [3.7882]
  + return ValidPathInfo(readNarInfo(storePath));
• edit in src/hydra-queue-runner/local-binary-cache.hh at line 3
  [3.9268]

  [3.9268]
  + #include "crypto.hh"
• edit in src/hydra-queue-runner/local-binary-cache.hh at line 7
  [3.9309]

  [3.9309]
  +

  + struct NarInfo;
• edit in src/hydra-queue-runner/local-binary-cache.hh at line 15
  [3.9416]

  [3.9416]
  +

  + std::unique_ptr<SecretKey> secretKey;

  + std::unique_ptr<PublicKeys> publicKeys;
• replacement in src/hydra-queue-runner/local-binary-cache.hh at line 21
  [3.9426]→[3.9426:9500](∅→∅)
  − LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir);
  [3.9426]

  [3.9500]
  + LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,

  + const Path & secretKeyFile, const Path & publicKeyFile);
• edit in src/hydra-queue-runner/local-binary-cache.hh at line 29
  [3.9630]→[3.9630:9712](∅→∅),[3.9712]→[2.1399:1441](∅→∅),[2.1441]→[3.9712:9719](∅→∅),[3.9712]→[3.9712:9719](∅→∅)
  −

  − struct NarInfo

  − {

  − ValidPathInfo info;

  − std::string narUrl;

  − std::string compression = "none";

  − };