S3BinaryCacheStore: Use disk cache

Dependencies

• [2] US3QQSHJ Updates for negative .narinfo caching
• [3] 24BMQDZA Start of single-process hydra-queue-runner
• [4] AMXZL5OR Use US standard S3 region
• [5] QSBS2ISO S3BinaryCacheStore::isValidPath(): Do a GET instead of HEAD
• [6] RQUAATWB Add status dump facility
• [7] GTUZLZRH Add an S3-backed binary cache store
• [8] XLYHZUHT Cache .narinfo lookups
• [9] SOB276BA Keep some statistics for the binary cache stores
• [10] RX6UB7YW Better AWS error messages
• [11] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [12] EYR3EW6J Keep stats for the Hydra auto scaler
• [13] BAFICF73 Support hydra-build-products on binary cache stores
• [14] DKJFD6JN Process Nix API changes
• [15] YHP5DSOO Improve parsing of hydra-build-products

Change contents

• replacement in src/hydra-queue-runner/build-result.cc at line 24
  [3.957]→[3.957:1073](∅→∅)
  − res.closureSize += info.narSize;

  − if (outputs.find(path) != outputs.end()) res.size += info.narSize;
  [3.957]

  [3.1073]
  + res.closureSize += info->narSize;

  + if (outputs.find(path) != outputs.end()) res.size += info->narSize;
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 632
  [3.2455]

  [3.1781]
  +

  + auto store = getDestStore();
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 635
  [3.1782]→[3.1782:1857](∅→∅)
  − auto store = dynamic_cast<S3BinaryCacheStore *>(&*getDestStore());
  [3.1782]

  [3.1857]
  + root.attr("store");

  + JSONObject nested(out);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 638
  [3.1858]→[3.1858:1947](∅→∅)
  − if (store) {

  − root.attr("store");

  − JSONObject nested(out);
  [3.1858]

  [3.1947]
  + auto & stats = store->getStats();

  + nested.attr("narInfoRead", stats.narInfoRead);

  + nested.attr("narInfoReadAverted", stats.narInfoReadAverted);

  + nested.attr("narInfoMissing", stats.narInfoMissing);

  + nested.attr("narInfoWrite", stats.narInfoWrite);

  + nested.attr("narInfoCacheSize", stats.pathInfoCacheSize);

  + nested.attr("narRead", stats.narRead);

  + nested.attr("narReadBytes", stats.narReadBytes);

  + nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);

  + nested.attr("narWrite", stats.narWrite);

  + nested.attr("narWriteAverted", stats.narWriteAverted);

  + nested.attr("narWriteBytes", stats.narWriteBytes);

  + nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);

  + nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);

  + nested.attr("narCompressionSavings",

  + stats.narWriteBytes

  + ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes

  + : 0.0);

  + nested.attr("narCompressionSpeed", // MiB/s

  + stats.narWriteCompressionTimeMs

  + ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)

  + : 0.0);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 661
  [3.1948]→[3.1948:2053](∅→∅),[3.2053]→[3.844:917](∅→∅),[3.917]→[2.0:65](∅→∅),[2.65]→[3.2053:2114](∅→∅),[3.917]→[3.2053:2114](∅→∅),[3.2053]→[3.2053:2114](∅→∅),[3.2114]→[3.918:987](∅→∅),[3.987]→[3.2114:2830](∅→∅),[3.2114]→[3.2114:2830](∅→∅)
  − auto & stats = store->getStats();

  − nested.attr("narInfoRead", stats.narInfoRead);

  − nested.attr("narInfoReadAverted", stats.narInfoReadAverted);

  − nested.attr("narInfoMissing", stats.narInfoMissing);

  − nested.attr("narInfoWrite", stats.narInfoWrite);

  − nested.attr("narInfoCacheSize", stats.narInfoCacheSize);

  − nested.attr("narRead", stats.narRead);

  − nested.attr("narReadBytes", stats.narReadBytes);

  − nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);

  − nested.attr("narWrite", stats.narWrite);

  − nested.attr("narWriteAverted", stats.narWriteAverted);

  − nested.attr("narWriteBytes", stats.narWriteBytes);

  − nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);

  − nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);

  − nested.attr("narCompressionSavings",

  − stats.narWriteBytes

  − ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
  [3.1948]

  [3.2830]
  + auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);

  + if (s3Store) {

  + nested.attr("s3");

  + JSONObject nested2(out);

  + auto & s3Stats = s3Store->getS3Stats();

  + nested2.attr("put", s3Stats.put);

  + nested2.attr("putBytes", s3Stats.putBytes);

  + nested2.attr("putTimeMs", s3Stats.putTimeMs);

  + nested2.attr("putSpeed",

  + s3Stats.putTimeMs

  + ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 673
  [3.2854]→[3.2854:3068](∅→∅)
  − nested.attr("narCompressionSpeed", // MiB/s

  − stats.narWriteCompressionTimeMs

  − ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
  [3.2854]

  [3.3068]
  + nested2.attr("get", s3Stats.get);

  + nested2.attr("getBytes", s3Stats.getBytes);

  + nested2.attr("getTimeMs", s3Stats.getTimeMs);

  + nested2.attr("getSpeed",

  + s3Stats.getTimeMs

  + ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 680
  [3.3092]→[3.3092:4388](∅→∅)
  −

  − auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);

  − if (s3Store) {

  − nested.attr("s3");

  − JSONObject nested2(out);

  − auto & s3Stats = s3Store->getS3Stats();

  − nested2.attr("put", s3Stats.put);

  − nested2.attr("putBytes", s3Stats.putBytes);

  − nested2.attr("putTimeMs", s3Stats.putTimeMs);

  − nested2.attr("putSpeed",

  − s3Stats.putTimeMs

  − ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)

  − : 0.0);

  − nested2.attr("get", s3Stats.get);

  − nested2.attr("getBytes", s3Stats.getBytes);

  − nested2.attr("getTimeMs", s3Stats.getTimeMs);

  − nested2.attr("getSpeed",

  − s3Stats.getTimeMs

  − ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)

  − : 0.0);

  − nested2.attr("head", s3Stats.head);

  − nested2.attr("costDollarApprox",

  − (s3Stats.get + s3Stats.head) / 10000.0 * 0.004

  − + s3Stats.put / 1000.0 * 0.005 +

  − + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);

  − }
  [3.3092]

  [3.4388]
  + nested2.attr("head", s3Stats.head);

  + nested2.attr("costDollarApprox",

  + (s3Stats.get + s3Stats.head) / 10000.0 * 0.004

  + + s3Stats.put / 1000.0 * 0.005 +

  + + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 4
  [3.64]

  [3.1334]
  + #include "nar-info-disk-cache.hh"
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 41
  [3.2347]

  [3.2347]
  + {

  + diskCache = getNarInfoDiskCache();

  + }

  +

  + std::string S3BinaryCacheStore::getUri()
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 47
  [3.2349]

  [3.2349]
  + return "s3://" + bucketName;
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 60
  [3.2621]→[3.2621:2905](∅→∅)
  − /* Create the bucket if it doesn't already exists. */

  − // FIXME: HeadBucket would be more appropriate, but doesn't return

  − // an easily parsed 404 message.

  − auto res = client->GetBucketLocation(

  − Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
  [3.2621]

  [3.2905]
  + if (!diskCache->cacheExists(getUri())) {
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 62
  [3.2906]→[3.2906:3014](∅→∅),[3.3014]→[3.141:259](∅→∅)
  − if (!res.IsSuccess()) {

  − if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)

  − throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
  [3.2906]

  [3.3095]
  + /* Create the bucket if it doesn't already exists. */

  + // FIXME: HeadBucket would be more appropriate, but doesn't return

  + // an easily parsed 404 message.

  + auto res = client->GetBucketLocation(

  + Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 68
  [3.3096]→[3.260:701](∅→∅)
  − checkAws(format("AWS error creating bucket ‘%s’") % bucketName,

  − client->CreateBucket(

  − Aws::S3::Model::CreateBucketRequest()

  − .WithBucket(bucketName)

  − .WithCreateBucketConfiguration(

  − Aws::S3::Model::CreateBucketConfiguration()

  − /* .WithLocationConstraint(

  − Aws::S3::Model::BucketLocationConstraint::US) */ )));
  [3.3096]

  [3.3443]
  + if (!res.IsSuccess()) {

  + if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)

  + throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());

  +

  + checkAws(format("AWS error creating bucket ‘%s’") % bucketName,

  + client->CreateBucket(

  + Aws::S3::Model::CreateBucketRequest()

  + .WithBucket(bucketName)

  + .WithCreateBucketConfiguration(

  + Aws::S3::Model::CreateBucketConfiguration()

  + /* .WithLocationConstraint(

  + Aws::S3::Model::BucketLocationConstraint::US) */ )));

  + }

  +

  + diskCache->createCache(getUri());
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 97
  [3.617]→[3.617:678](∅→∅)
  − bool S3BinaryCacheStore::isValidPath(const Path & storePath)
  [3.617]

  [3.678]
  + bool S3BinaryCacheStore::isValidPathUncached(const Path & storePath)
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 100
  [3.690]→[3.690:722](∅→∅)
  − readNarInfo(storePath);
  [3.690]

  [3.722]
  + queryPathInfo(storePath);
• edit in src/hydra-queue-runner/s3-binary-cache-store.hh at line 27
  [3.6068]

  [3.6068]
  +

  + std::string getUri();
• replacement in src/hydra-queue-runner/s3-binary-cache-store.hh at line 43
  [3.861]→[3.861:916](∅→∅)
  − bool isValidPath(const Path & storePath) override;
  [3.861]

  [3.5862]
  + bool isValidPathUncached(const Path & storePath) override;