Keep some statistics for the binary cache stores

Dependencies

• [2] ZM34T2NW Typo
• [3] EOO4EFWD Use a single BinaryCacheStore for all threads
• [4] LE4VZIY5 More stats
• [5] O64P4XJS Keep per-machine stats
• [6] IK2UBDAU Revive jobset scheduling
• [7] H7SZRHUB Use nix::willBuildLocally()
• [8] 5AIYUMTB Basic remote building
• [9] 63W4T5PU hydra-queue-runner: More stats
• [10] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
• [11] EYR3EW6J Keep stats for the Hydra auto scaler
• [12] GTUZLZRH Add an S3-backed binary cache store
• [13] ZTVVUMTK JSONObject doesn't handle 64-bit integers
• [14] DWFTK56E Keep track of how many threads are waiting
• [15] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [16] RQUAATWB Add status dump facility
• [17] SODOV2CM Automatically reload $NIX_REMOTE_SYSTEMS when it changes
• [18] 24BMQDZA Start of single-process hydra-queue-runner
• [19] 32HHP5CW hydra-queue-runner: Support generating a signed binary cache
• [20] N2NKSKHS Refactor local binary cache code into a subclass
• [21] A2GL5FOZ Moar stats
• [*] 3FQ65IXO hydra-queue-runner: Compress binary cache NARs using xz
• [*] K5G5GZY7 Guard against concurrent invocations of hydra-queue-runner

Change contents

• edit in src/hydra-queue-runner/binary-cache-store.cc at line 35
  [4.2400]

  [4.358]
  + const BinaryCacheStore::Stats & BinaryCacheStore::getStats()

  + {

  + return stats;

  + }

  +
• edit in src/hydra-queue-runner/binary-cache-store.cc at line 68
  [4.405]

  [4.405]
  + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
• replacement in src/hydra-queue-runner/binary-cache-store.cc at line 72
  [4.622]→[4.622:709](∅→∅)
  − % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
  [4.622]

  [4.3170]
  + % duration);
• replacement in src/hydra-queue-runner/binary-cache-store.cc at line 76
  [4.1349]→[4.598:664](∅→∅)
  − if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz);
  [4.1349]

  [4.3385]
  + if (!fileExists(narInfo.url)) {

  + stats.narWrite++;

  + upsertFile(narInfo.url, narXz);

  + } else

  + stats.narWriteAverted++;

  +

  + stats.narWriteBytes += nar.size();

  + stats.narWriteCompressedBytes += narXz.size();

  + stats.narWriteCompressionTimeMs += duration;
• edit in src/hydra-queue-runner/binary-cache-store.cc at line 90
  [4.715]

  [4.4219]
  +

  + stats.narInfoWrite++;
• edit in src/hydra-queue-runner/binary-cache-store.cc at line 96
  [4.4304]

  [4.716]
  + stats.narInfoRead++;

  +
• edit in src/hydra-queue-runner/binary-cache-store.cc at line 122
  [4.914]

  [23.1104]
  +

  + stats.narRead++;

  + stats.narReadCompressedBytes += nar.size();
• edit in src/hydra-queue-runner/binary-cache-store.cc at line 135
  [4.6029]

  [4.6029]
  + stats.narReadBytes += nar.size();

  +
• edit in src/hydra-queue-runner/binary-cache-store.hh at line 5
  [4.9292]

  [4.9292]
  +

  + #include <atomic>
• edit in src/hydra-queue-runner/binary-cache-store.hh at line 40
  [4.1258]

  [4.1258]
  +

  + struct Stats

  + {

  + std::atomic<uint64_t> narInfoRead{0};

  + std::atomic<uint64_t> narInfoWrite{0};

  + std::atomic<uint64_t> narRead{0};

  + std::atomic<uint64_t> narReadBytes{0};

  + std::atomic<uint64_t> narReadCompressedBytes{0};

  + std::atomic<uint64_t> narWrite{0};

  + std::atomic<uint64_t> narWriteAverted{0};

  + std::atomic<uint64_t> narWriteBytes{0};

  + std::atomic<uint64_t> narWriteCompressedBytes{0};

  + std::atomic<uint64_t> narWriteCompressionTimeMs{0};

  + };
• edit in src/hydra-queue-runner/binary-cache-store.hh at line 55
  [4.1259]

  [4.9501]
  + const Stats & getStats();

  +
• edit in src/hydra-queue-runner/binary-cache-store.hh at line 59
  [4.9511]

  [4.1260]
  + Stats stats;

  +
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 26
  [4.7892]→[3.482:979](∅→∅)
  −

  − #if 0

  − auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),

  − "/home/eelco/Misc/Keys/test.nixos.org/secret",

  − "/home/eelco/Misc/Keys/test.nixos.org/public",

  − "/tmp/binary-cache");

  − #endif

  −

  − auto store = std::make_shared<S3BinaryCacheStore>(

  − []() { return openStore(); },

  − "/home/eelco/Misc/Keys/test.nixos.org/secret",

  − "/home/eelco/Misc/Keys/test.nixos.org/public",

  − "nix-test-cache-3");;

  − store->init();

  − _destStore = store;
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 520
  [4.305]→[4.0:108](∅→∅),[4.818]→[4.0:108](∅→∅)
  − root.attr("bytesSent"); out << bytesSent;

  − root.attr("bytesReceived"); out << bytesReceived;
  [4.305]

  [4.1221]
  + root.attr("bytesSent", bytesSent);

  + root.attr("bytesReceived", bytesReceived);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 530
  [4.631]→[4.667:841](∅→∅),[4.667]→[4.667:841](∅→∅)
  − root.attr("avgStepTime"); out << (float) totalStepTime / nrStepsDone;

  − root.attr("avgStepBuildTime"); out << (float) totalStepBuildTime / nrStepsDone;
  [4.631]

  [4.841]
  + root.attr("avgStepTime", (float) totalStepTime / nrStepsDone);

  + root.attr("avgStepBuildTime", (float) totalStepBuildTime / nrStepsDone);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 553
  [4.4893]→[4.4893:5101](∅→∅)
  − nested2.attr("avgStepTime"); out << (float) s->totalStepTime / s->nrStepsDone;

  − nested2.attr("avgStepBuildTime"); out << (float) s->totalStepBuildTime / s->nrStepsDone;
  [4.4893]

  [4.1087]
  + nested2.attr("avgStepTime", (float) s->totalStepTime / s->nrStepsDone);

  + nested2.attr("avgStepBuildTime", (float) s->totalStepBuildTime / s->nrStepsDone);
• replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 565
  [4.2017]→[4.2017:2095](∅→∅)
  − nested2.attr("shareUsed"); out << jobset.second->shareUsed();
  [4.2017]

  [4.2095]
  + nested2.attr("shareUsed", jobset.second->shareUsed());
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 585
  [4.2455]

  [4.401]
  +

  + auto store = dynamic_cast<S3BinaryCacheStore *>(&*getDestStore());

  +

  + if (store) {

  + root.attr("store");

  + JSONObject nested(out);

  +

  + auto & stats = store->getStats();

  + nested.attr("narInfoRead", stats.narInfoRead);

  + nested.attr("narInfoWrite", stats.narInfoWrite);

  + nested.attr("narRead", stats.narRead);

  + nested.attr("narReadBytes", stats.narReadBytes);

  + nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);

  + nested.attr("narWrite", stats.narWrite);

  + nested.attr("narWriteAverted", stats.narWriteAverted);

  + nested.attr("narWriteBytes", stats.narWriteBytes);

  + nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);

  + nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);

  + nested.attr("narCompressionSavings",

  + stats.narWriteBytes

  + ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes

  + : 0.0);

  + nested.attr("narCompressionSpeed", // MiB/s

  + stats.narWriteCompressionTimeMs

  + ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)

  + : 0.0);

  +

  + auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);

  + if (s3Store) {

  + nested.attr("s3");

  + JSONObject nested2(out);

  + auto & s3Stats = s3Store->getS3Stats();

  + nested2.attr("put", s3Stats.put);

  + nested2.attr("putBytes", s3Stats.putBytes);

  + nested2.attr("putTimeMs", s3Stats.putTimeMs);

  + nested2.attr("putSpeed",

  + s3Stats.putTimeMs

  + ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)

  + : 0.0);

  + nested2.attr("get", s3Stats.get);

  + nested2.attr("getBytes", s3Stats.getBytes);

  + nested2.attr("getTimeMs", s3Stats.getTimeMs);

  + nested2.attr("getSpeed",

  + s3Stats.getTimeMs

  + ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)

  + : 0.0);

  + nested2.attr("head", s3Stats.head);

  + nested2.attr("costDollarApprox",

  + (s3Stats.get + s3Stats.head) / 10000.0 * 0.004

  + + s3Stats.put / 1000.0 * 0.005 +

  + + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);

  + }

  + }
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 725
  [24.643]

  [24.643]
  +

  + #if 0

  + auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),

  + "/home/eelco/Misc/Keys/test.nixos.org/secret",

  + "/home/eelco/Misc/Keys/test.nixos.org/public",

  + "/tmp/binary-cache");

  + #endif

  +

  + auto store = std::make_shared<S3BinaryCacheStore>(

  + []() { return openStore(); },

  + "/home/eelco/Misc/Keys/test.nixos.org/secret",

  + "/home/eelco/Misc/Keys/test.nixos.org/public",

  + "nix-test-cache-3");;

  + store->init();

  + _destStore = store;
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 63
  [4.3480]

  [4.3480]
  + }

  +

  + const S3BinaryCacheStore::Stats & S3BinaryCacheStore::getS3Stats()

  + {

  + return stats;
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 72
  [4.3547]

  [4.3547]
  + stats.head++;

  +
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 101
  [4.4313]

  [4.4313]
  + stats.put++;

  + stats.putBytes += data.size();

  +
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 109
  [4.4471]

  [4.4471]
  +

  + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 113
  [4.4555]→[4.4555:4692](∅→∅)
  − % bucketName % path

  − % data.size()

  − % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
  [4.4555]

  [4.4692]
  + % bucketName % path % data.size() % duration);

  +

  + stats.putTimeMs += duration;
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 129
  [4.4996]

  [4.4996]
  + stats.get++;

  +
• edit in src/hydra-queue-runner/s3-binary-cache-store.cc at line 138
  [4.5229]

  [4.5229]
  +

  + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
• replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 142
  [2.85]→[4.5309:5445](∅→∅),[4.5309]→[4.5309:5445](∅→∅)
  − % bucketName % path

  − % res.size()

  − % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
  [2.85]

  [4.5445]
  + % bucketName % path % res.size() % duration);

  +

  + stats.getBytes += res.size();

  + stats.getTimeMs += duration;
• edit in src/hydra-queue-runner/s3-binary-cache-store.hh at line 4
  [4.5567]

  [4.5567]
  +

  + #include <atomic>
• edit in src/hydra-queue-runner/s3-binary-cache-store.hh at line 29
  [4.6069]

  [4.6069]
  + struct Stats

  + {

  + std::atomic<uint64_t> put{0};

  + std::atomic<uint64_t> putBytes{0};

  + std::atomic<uint64_t> putTimeMs{0};

  + std::atomic<uint64_t> get{0};

  + std::atomic<uint64_t> getBytes{0};

  + std::atomic<uint64_t> getTimeMs{0};

  + std::atomic<uint64_t> head{0};

  + };

  +

  + const Stats & getS3Stats();

  +
• edit in src/hydra-queue-runner/s3-binary-cache-store.hh at line 43
  [4.6078]

  [4.6078]
  +

  + Stats stats;