Sync with Nix

Dependencies

• [2] QSBS2ISO S3BinaryCacheStore::isValidPath(): Do a GET instead of HEAD
• [3] RITB7JRK After uploading a .narinfo, add it to the LRU cache
• [4] KS7NNPQW Bleh Automake
• [5] B2L4T3X6 Sync with Nix
• [6] ENXUSMSV Make concurrency more robust
• [7] 7LB6QBXY Keep track of the number of build steps that are being built
• [8] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [9] GTUZLZRH Add an S3-backed binary cache store
• [10] HJOEIMLR Refactor
• [11] 32HHP5CW hydra-queue-runner: Support generating a signed binary cache
• [12] W2AOTSS6 Rename class
• [13] MB3TISH2 Rate-limit the number of threads copying closures at the same time
• [14] 5AIYUMTB Basic remote building
• [15] GH4S4AWM Rename file
• [16] EOO4EFWD Use a single BinaryCacheStore for all threads
• [17] SOB276BA Keep some statistics for the binary cache stores
• [18] XLYHZUHT Cache .narinfo lookups
• [19] N2NKSKHS Refactor local binary cache code into a subclass
• [20] MBZRPJTZ Fix log message
• [21] GJV2J5HX Pool local store connections
• [22] Y5LMEC35 BinaryCacheStore: Respect build-use-substitutes
• [23] 3FQ65IXO hydra-queue-runner: Compress binary cache NARs using xz
• [24] MHVIT4JY Split hydra-queue-runner.cc more
• [25] N4IROACV Move buildRemote() into State
• [26] YZAI5GQU Implement a database connection pool
• [27] 24BMQDZA Start of single-process hydra-queue-runner

Change contents

• file deletion: binary-cache-store.cc (----------)local-binary-cache.cc (----------)
  [6.187]→[6.46:91](∅→∅),[6.91]→[6.1879:1879](∅→∅),[6.187]→[6.9206:9251](∅→∅),[6.9251]→[6.1879:1879](∅→∅)
  −

  − #include "archive.hh"

  − #include "derivations.hh"

  − #include "globals.hh"

  − #include "worker-protocol.hh"

  −

  − namespace nix {

  −

  − const Path & secretKeyFile, const Path & publicKeyFile)

  − {

  − if (secretKeyFile != "")

  − secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));

  −

  − if (publicKeyFile != "") {

  − publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);

  − auto key = PublicKey(readFile(publicKeyFile));

  − publicKeys->emplace(key.name, key);

  − }

  − }

  −

  − {

  − }

  −

  − {

  − }

  −

  − const string & nar)

  − {

  −

  −

  −

  − throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);

  −

  − /* Compress the NAR. */

  − auto now1 = std::chrono::steady_clock::now();

  − narInfo->compression = "xz";

  − string narXz = compressXZ(nar);

  − auto now2 = std::chrono::steady_clock::now();

  −

  − printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")

  − % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)

  −

  − /* Atomically write the NAR file. */

  −

  − /* Atomically write the NAR info file.*/

  −

  − }

  −

  − {

  −

  − if (publicKeys) {

  − throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);

  − }

  −

  − }

  −

  − {

  − }

  −

  − {

  − assert(!sign);

  −

  − auto res = readNarInfo(storePath);

  −

  −

  − /* Decompress the NAR. FIXME: would be nice to have the remote

  − side do this. */

  − if (res.compression == "none")

  − ;

  − else if (res.compression == "xz")

  − nar = decompressXZ(nar);

  − else

  − throw Error(format("unknown NAR compression type ‘%1%’") % nar);

  −

  − printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());

  −

  − assert(nar.size() % 8 == 0);

  −

  − sink((unsigned char *) nar.c_str(), nar.size());

  −

  − // FIXME: check integrity of NAR.

  −

  − sink << exportMagic << storePath << res.references << res.deriver << 0;

  − }

  −

  − {

  − assert(!requireSignature);

  − Paths res;

  − while (true) {

  − unsigned long long n = readLongLong(source);

  − if (n == 0) break;

  − if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");

  − res.push_back(importPath(source));

  − }

  − return res;

  − }

  −

  − struct TeeSource : Source

  − {

  − Source & readSource;

  − std::string data;

  − TeeSource(Source & readSource) : readSource(readSource)

  − {

  − }

  − size_t read(unsigned char * data, size_t len)

  − {

  − size_t n = readSource.read(data, len);

  − this->data.append((char *) data, n);

  − return n;

  − }

  − };

  −

  − struct NopSink : ParseSink

  − {

  − };

  −

  − {

  − /* FIXME: some cut&paste of LocalStore::importPath(). */

  −

  − /* Extract the NAR from the source. */

  − TeeSource tee(source);

  − NopSink sink;

  − parseDump(sink, tee);

  −

  − uint32_t magic = readInt(source);

  − if (magic != exportMagic)

  − throw Error("Nix archive cannot be imported; wrong format");

  −

  − ValidPathInfo info;

  − info.path = readStorePath(source);

  −

  − info.references = readStorePaths<PathSet>(source);

  −

  − readString(source); // deriver, don't care

  −

  − bool haveSignature = readInt(source) == 1;

  − assert(!haveSignature);

  −

  − addToCache(info, tee.data);

  −

  − return info.path;

  − }

  −

  − {

  − return ValidPathInfo(readNarInfo(storePath));

  − }

  −

  − SubstitutablePathInfos & infos)

  − {

  − PathSet left;

  −

  − for (auto & storePath : paths) {

  − left.insert(storePath);

  − continue;

  − }

  − SubstitutablePathInfo sub;

  − sub.references = info.references;

  − sub.downloadSize = 0;

  − sub.narSize = info.narSize;

  − infos.emplace(storePath, sub);

  − }

  −

  − }

  −

  − {

  − for (auto & storePath : paths) {

  − assert(!isDerivation(storePath));

  −

  − if (isValidPath(storePath)) continue;

  −

  −

  −

  −

  − for (auto & ref : info.references)

  − if (ref != storePath)

  − ensurePath(ref);

  −

  − StringSink sink;

  − dumpPath(storePath, sink);

  −

  − addToCache(info, sink.s);

  − }

  − }

  −

  − {

  − buildPaths({path});

  − }

  −

  − }

  − void BinaryCacheStore::ensurePath(const Path & path)

  − ValidPathInfo info = localStore->queryPathInfo(storePath);

  −

  − if (!localStore->isValidPath(storePath))

  − localStore->ensurePath(storePath);

  − localStore->addTempRoot(storePath);

  − if (!localStore)

  − throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath);

  − void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)

  − if (settings.useSubstitutes)

  − localStore->querySubstitutablePathInfos(left, infos);

  − ValidPathInfo info = localStore->queryPathInfo(storePath);

  − if (!localStore->isValidPath(storePath)) {

  −

  − if (!localStore) return;

  − void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,

  − ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)

  − Path BinaryCacheStore::importPath(Source & source)

  − Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)

  − stats.narReadBytes += nar.size();

  −

  − auto nar = getFile(res.url);

  −

  − stats.narRead++;

  − stats.narReadCompressedBytes += nar.size();

  − void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)

  − return fileExists(narInfoFileFor(storePath));

  − bool BinaryCacheStore::isValidPath(const Path & storePath)

  − {

  − auto state_(state.lock());

  − state_->narInfoCache.upsert(storePath, narInfo);

  − stats.narInfoCacheSize = state_->narInfoCache.size();

  − }

  −

  − return *narInfo;

  − if (!narInfo->checkSignature(*publicKeys))

  −

  − auto narInfoFile = narInfoFileFor(storePath);

  − auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);

  − assert(narInfo->path == storePath);

  −

  − stats.narInfoRead++;

  − {

  − auto state_(state.lock());

  − auto res = state_->narInfoCache.get(storePath);

  − if (res) {

  − stats.narInfoReadAverted++;

  − return **res;

  − }

  − }

  − NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)

  −

  − stats.narInfoWrite++;

  − upsertFile(narInfoFile, narInfo->to_string());

  −

  − {

  − auto state_(state.lock());

  − state_->narInfoCache.upsert(narInfo->path, narInfo);

  − stats.narInfoCacheSize = state_->narInfoCache.size();

  − }

  − if (secretKey) narInfo->sign(*secretKey);

  − stats.narWrite++;

  − } else

  − stats.narWriteAverted++;

  −

  − stats.narWriteBytes += nar.size();

  − stats.narWriteCompressedBytes += narXz.size();

  − stats.narWriteCompressionTimeMs += duration;

  − upsertFile(narInfo->url, narXz);

  − narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";

  − if (!fileExists(narInfo->url)) {

  − % duration);

  − % narInfo->path % narInfo->narSize

  − auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

  − narInfo->fileHash = hashString(htSHA256, narXz);

  − narInfo->fileSize = narXz.size();

  − if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)

  − narInfo->narSize = nar.size();

  − narInfo->narHash = hashString(htSHA256, nar);

  − auto narInfo = make_ref<NarInfo>(info);

  − auto narInfoFile = narInfoFileFor(info.path);

  − if (fileExists(narInfoFile)) return;

  − void BinaryCacheStore::addToCache(const ValidPathInfo & info,

  − assertStorePath(storePath);

  − return storePathToHash(storePath) + ".narinfo";

  − Path BinaryCacheStore::narInfoFileFor(const Path & storePath)

  − const BinaryCacheStore::Stats & BinaryCacheStore::getStats()

  − {

  − return stats;

  − }

  −

  − std::string cacheInfoFile = "nix-cache-info";

  − if (!fileExists(cacheInfoFile))

  − upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");

  − void BinaryCacheStore::init()

  − : localStore(localStore)

  − BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,

  −

  − #include <chrono>

  − #include "nar-info.hh"

  − #include "compression.hh"

  − #include "binary-cache-store.hh"

  − #include "sync.hh"
• file deletion: binary-cache-store.hh (----------)local-binary-cache.hh (----------)
  [6.187]→[6.125:170](∅→∅),[6.170]→[6.9253:9253](∅→∅),[6.187]→[6.12234:12279](∅→∅),[6.12279]→[6.9253:9253](∅→∅)
  − #pragma once

  −

  − #include "store-api.hh"

  −

  − namespace nix {

  −

  − {

  − private:

  −

  − std::unique_ptr<SecretKey> secretKey;

  − std::unique_ptr<PublicKeys> publicKeys;

  −

  −

  − const Path & secretKeyFile, const Path & publicKeyFile);

  −

  − virtual bool fileExists(const std::string & path) = 0;

  −

  − virtual void upsertFile(const std::string & path, const std::string & data) = 0;

  −

  − virtual std::string getFile(const std::string & path) = 0;

  −

  − public:

  −

  − private:

  −

  −

  − void addToCache(const ValidPathInfo & info, const string & nar);

  −

  − NarInfo readNarInfo(const Path & storePath);

  −

  − public:

  −

  − bool isValidPath(const Path & path) override;

  −

  − PathSet queryValidPaths(const PathSet & paths) override

  − { abort(); }

  −

  − PathSet queryAllValidPaths() override

  − { abort(); }

  −

  − ValidPathInfo queryPathInfo(const Path & path) override;

  −

  − Hash queryPathHash(const Path & path) override

  − { abort(); }

  −

  − void queryReferrers(const Path & path,

  − PathSet & referrers) override

  − { abort(); }

  −

  − Path queryDeriver(const Path & path) override

  − { abort(); }

  −

  − PathSet queryValidDerivers(const Path & path) override

  − { abort(); }

  −

  − PathSet queryDerivationOutputs(const Path & path) override

  − { abort(); }

  −

  − StringSet queryDerivationOutputNames(const Path & path) override

  − { abort(); }

  −

  − Path queryPathFromHashPart(const string & hashPart) override

  − { abort(); }

  −

  − PathSet querySubstitutablePaths(const PathSet & paths) override

  − { abort(); }

  −

  − void querySubstitutablePathInfos(const PathSet & paths,

  − SubstitutablePathInfos & infos) override;

  −

  − Path addToStore(const string & name, const Path & srcPath,

  − bool recursive = true, HashType hashAlgo = htSHA256,

  − PathFilter & filter = defaultPathFilter, bool repair = false) override

  − { abort(); }

  −

  − Path addTextToStore(const string & name, const string & s,

  − const PathSet & references, bool repair = false) override

  − { abort(); }

  −

  − void exportPath(const Path & path, bool sign,

  − Sink & sink) override;

  −

  − Paths importPaths(bool requireSignature, Source & source) override;

  −

  − Path importPath(Source & source);

  −

  − void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;

  −

  − BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,

  − BuildMode buildMode = bmNormal) override

  − { abort(); }

  −

  − void ensurePath(const Path & path) override;

  −

  − void addTempRoot(const Path & path) override

  − { abort(); }

  −

  − void addIndirectRoot(const Path & path) override

  − { abort(); }

  −

  − void syncWithGC() override

  − { }

  −

  − Roots findRoots() override

  − { abort(); }

  −

  − void collectGarbage(const GCOptions & options, GCResults & results) override

  − { abort(); }

  −

  − PathSet queryFailedPaths() override

  − { return PathSet(); }

  −

  − void clearFailedPaths(const PathSet & paths) override

  − { }

  −

  − void optimiseStore() override

  − { }

  −

  − bool verifyStore(bool checkContents, bool repair) override

  − { return true; }

  −

  − };

  −

  − }

  −

  − protected:

  − std::string narInfoFileFor(const Path & storePath);

  − Stats stats;

  −

  − virtual void init();

  −

  − const Stats & getStats();

  −

  −

  − struct Stats

  − {

  − std::atomic<uint64_t> narInfoRead{0};

  − std::atomic<uint64_t> narInfoWrite{0};

  − std::atomic<uint64_t> narRead{0};

  − std::atomic<uint64_t> narReadBytes{0};

  − std::atomic<uint64_t> narReadCompressedBytes{0};

  − std::atomic<uint64_t> narWrite{0};

  − std::atomic<uint64_t> narWriteAverted{0};

  − std::atomic<uint64_t> narWriteBytes{0};

  − std::atomic<uint64_t> narWriteCompressedBytes{0};

  − std::atomic<uint64_t> narWriteCompressionTimeMs{0};

  − };

  − std::atomic<uint64_t> narInfoCacheSize{0};

  − std::atomic<uint64_t> narInfoReadAverted{0};

  − BinaryCacheStore(std::shared_ptr<Store> localStore,

  −

  − struct State

  − {

  − LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024};

  − };

  −

  − Sync<State> state;

  −

  − protected:

  − std::shared_ptr<Store> localStore;

  − class BinaryCacheStore : public Store

  −

  − struct NarInfo;

  −

  − #include <atomic>

  −

  − #include "lru-cache.hh"

  − #include "sync.hh"

  − #include "pool.hh"

  − #include "crypto.hh"
• file deletion: local-binary-cache-store.cc (----------)
  [6.187]→[6.2619:2670](∅→∅),[6.2670]→[6.1463:1463](∅→∅)
  − #include "local-binary-cache-store.hh"

  −

  − namespace nix {

  −

  − const Path & secretKeyFile, const Path & publicKeyFile,

  − const Path & binaryCacheDir)

  − , binaryCacheDir(binaryCacheDir)

  − {

  − }

  −

  − void LocalBinaryCacheStore::init()

  − {

  − createDirs(binaryCacheDir + "/nar");

  − BinaryCacheStore::init();

  − }

  −

  − static void atomicWrite(const Path & path, const std::string & s)

  − {

  − Path tmp = path + ".tmp." + std::to_string(getpid());

  − AutoDelete del(tmp, false);

  − writeFile(tmp, s);

  − if (rename(tmp.c_str(), path.c_str()))

  − throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);

  − del.cancel();

  − }

  −

  − bool LocalBinaryCacheStore::fileExists(const std::string & path)

  − {

  − return pathExists(binaryCacheDir + "/" + path);

  − }

  −

  − void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)

  − {

  − atomicWrite(binaryCacheDir + "/" + path, data);

  − }

  −

  − std::string LocalBinaryCacheStore::getFile(const std::string & path)

  − {

  − return readFile(binaryCacheDir + "/" + path);

  − }

  −

  − }

  − : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)

  − LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
• file deletion: lru-cache.hh (----------)
  [6.187]→[6.2826:2862](∅→∅),[6.2862]→[6.989:989](∅→∅)
  − #pragma once

  −

  − #include <map>

  − #include <list>

  −

  − /* A simple least-recently used cache. Not thread-safe. */

  − template<typename Key, typename Value>

  − class LRUCache

  − {

  − private:

  −

  − size_t maxSize;

  −

  − // Stupid wrapper to get around circular dependency between Data

  − // and LRU.

  − struct LRUIterator;

  −

  − using Data = std::map<Key, std::pair<LRUIterator, Value>>;

  − using LRU = std::list<typename Data::iterator>;

  −

  − struct LRUIterator { typename LRU::iterator it; };

  −

  − Data data;

  − LRU lru;

  −

  − public:

  −

  − LRUCache(size_t maxSize) : maxSize(maxSize) { }

  −

  − /* Insert or upsert an item in the cache. */

  − void upsert(const Key & key, const Value & value)

  − {

  − erase(key);

  −

  − if (data.size() >= maxSize) {

  − /* Retire the oldest item. */

  − auto oldest = lru.begin();

  − data.erase(*oldest);

  − lru.erase(oldest);

  − }

  −

  − auto res = data.emplace(key, std::make_pair(LRUIterator(), value));

  − assert(res.second);

  − auto & i(res.first);

  −

  − auto j = lru.insert(lru.end(), i);

  −

  − i->second.first.it = j;

  − }

  −

  − bool erase(const Key & key)

  − {

  − auto i = data.find(key);

  − if (i == data.end()) return false;

  − lru.erase(i->second.first.it);

  − data.erase(i);

  − return true;

  − }

  −

  − /* Look up an item in the cache. If it's exists, it becomes the

  − most recently used item. */

  − // FIXME: use boost::optional?

  − Value * get(const Key & key)

  − {

  − auto i = data.find(key);

  − if (i == data.end()) return 0;

  −

  − /* Move this item to the back of the LRU list. */

  − lru.erase(i->second.first.it);

  − auto j = lru.insert(lru.end(), i);

  − i->second.first.it = j;

  −

  − return &i->second.second;

  − }

  −

  − size_t size()

  − {

  − return data.size();

  − }

  − };
• file deletion: local-binary-cache-store.hh (----------)
  [6.187]→[6.3230:3281](∅→∅),[6.3281]→[6.2672:2672](∅→∅)
  − #pragma once

  −

  − #include "binary-cache-store.hh"

  −

  − namespace nix {

  −

  − class LocalBinaryCacheStore : public BinaryCacheStore

  − {

  − private:

  −

  − Path binaryCacheDir;

  −

  − public:

  −

  −

  − void init() override;

  −

  − protected:

  −

  − bool fileExists(const std::string & path) override;

  −

  − void upsertFile(const std::string & path, const std::string & data) override;

  −

  − std::string getFile(const std::string & path) override;

  −

  − };

  −

  − }

  − const Path & secretKeyFile, const Path & publicKeyFile,

  − const Path & binaryCacheDir);

  − LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
• replacement in src/hydra-queue-runner/Makefile.am at line 5
  [6.131]→[6.0:77](∅→∅),[6.77]→[6.0:47](∅→∅),[6.47]→[6.119:178](∅→∅),[6.178]→[4.0:67](∅→∅)
  − build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \

  − binary-cache-store.hh binary-cache-store.cc \

  − local-binary-cache-store.hh local-binary-cache-store.cc \

  − s3-binary-cache-store.hh s3-binary-cache-store.cc \

  − lru-cache.hh
  [6.131]

  [6.322]
  + build-result.hh counter.hh token-server.hh state.hh db.hh \

  + s3-binary-cache-store.hh s3-binary-cache-store.cc