Remove s3binarystore (moved to nix in d155d80)

Dependencies

• [2] FS3HUMVU Fix Makefile.am
• [3] XO774HEK Fix a boost format string abort
• [4] DXFMYCNW Fix build
• [5] W4W4ZTNV Some more logging
• [6] 2PQPKMG7 S3BinaryCacheStore: Use disk cache
• [7] XLYHZUHT Cache .narinfo lookups
• [8] 24BMQDZA Start of single-process hydra-queue-runner
• [9] GH4S4AWM Rename file
• [10] 7LB6QBXY Keep track of the number of build steps that are being built
• [11] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
• [12] QSBS2ISO S3BinaryCacheStore::isValidPath(): Do a GET instead of HEAD
• [13] V6H6BWMK Sync with Nix
• [14] 5AIYUMTB Basic remote building
• [15] AMXZL5OR Use US standard S3 region
• [16] SOB276BA Keep some statistics for the binary cache stores
• [17] EOO4EFWD Use a single BinaryCacheStore for all threads
• [18] N2NKSKHS Refactor local binary cache code into a subclass
• [19] DIEY5USN Keep better bytesReceived/bytesSent stats
• [20] YZAI5GQU Implement a database connection pool
• [21] B2L4T3X6 Sync with Nix
• [22] ENXUSMSV Make concurrency more robust
• [23] MB3TISH2 Rate-limit the number of threads copying closures at the same time
• [24] GTUZLZRH Add an S3-backed binary cache store
• [25] N4IROACV Move buildRemote() into State
• [26] MHVIT4JY Split hydra-queue-runner.cc more
• [27] QDEGRYZV Reduce severity level of some message
• [28] RX6UB7YW Better AWS error messages
• [29] ZM34T2NW Typo
• [30] HJOEIMLR Refactor
• [31] US3QQSHJ Updates for negative .narinfo caching

Change contents

• file deletion: s3-binary-cache-store.cc (----------)
  [7.187]→[7.5469:5517](∅→∅),[7.5517]→[7.1297:1297](∅→∅)
  − #include "s3-binary-cache-store.hh"

  −

  − #include <aws/core/client/ClientConfiguration.h>

  − #include <aws/s3/S3Client.h>

  − #include <aws/s3/model/CreateBucketRequest.h>

  − #include <aws/s3/model/GetBucketLocationRequest.h>

  − #include <aws/s3/model/GetObjectRequest.h>

  − #include <aws/s3/model/HeadObjectRequest.h>

  − #include <aws/s3/model/PutObjectRequest.h>

  −

  − namespace nix {

  −

  − /* Helper: given an Outcome<R, E>, return R in case of success, or

  − throw an exception in case of an error. */

  − template<typename R, typename E>

  − R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)

  − {

  − if (!outcome.IsSuccess())

  − throw S3Error(

  − outcome.GetError().GetErrorType(),

  − fs.s + ": " + outcome.GetError().GetMessage());

  − return outcome.GetResultWithOwnership();

  − }

  −

  − S3BinaryCacheStore::S3BinaryCacheStore(std::shared_ptr<Store> localStore,

  − const Path & secretKeyFile, const std::string & bucketName)

  − : BinaryCacheStore(localStore, secretKeyFile)

  − , bucketName(bucketName)

  − , config(makeConfig())

  − , client(make_ref<Aws::S3::S3Client>(*config))

  − {

  − diskCache = getNarInfoDiskCache();

  − }

  −

  − std::string S3BinaryCacheStore::getUri()

  − {

  − return "s3://" + bucketName;

  − }

  −

  − ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()

  − {

  − auto res = make_ref<Aws::Client::ClientConfiguration>();

  − res->region = Aws::Region::US_EAST_1;

  − res->requestTimeoutMs = 600 * 1000;

  − return res;

  − }

  −

  − void S3BinaryCacheStore::init()

  − {

  − if (!diskCache->cacheExists(getUri())) {

  −

  − /* Create the bucket if it doesn't already exists. */

  − // FIXME: HeadBucket would be more appropriate, but doesn't return

  − // an easily parsed 404 message.

  − auto res = client->GetBucketLocation(

  − Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

  −

  − if (!res.IsSuccess()) {

  − if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)

  − throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());

  −

  − checkAws(format("AWS error creating bucket ‘%s’") % bucketName,

  − client->CreateBucket(

  − Aws::S3::Model::CreateBucketRequest()

  − .WithBucket(bucketName)

  − .WithCreateBucketConfiguration(

  − Aws::S3::Model::CreateBucketConfiguration()

  − /* .WithLocationConstraint(

  − Aws::S3::Model::BucketLocationConstraint::US) */ )));

  − }

  −

  − diskCache->createCache(getUri());

  − }

  −

  − BinaryCacheStore::init();

  − }

  −

  − const S3BinaryCacheStore::Stats & S3BinaryCacheStore::getS3Stats()

  − {

  − return stats;

  − }

  −

  − /* This is a specialisation of isValidPath() that optimistically

  − fetches the .narinfo file, rather than first checking for its

  − existence via a HEAD request. Since .narinfos are small, doing a

  − GET is unlikely to be slower than HEAD. */

  − bool S3BinaryCacheStore::isValidPathUncached(const Path & storePath)

  − {

  − try {

  − queryPathInfo(storePath);

  − return true;

  − } catch (InvalidPath & e) {

  − return false;

  − }

  − }

  −

  − bool S3BinaryCacheStore::fileExists(const std::string & path)

  − {

  − stats.head++;

  −

  − auto res = client->HeadObject(

  − Aws::S3::Model::HeadObjectRequest()

  − .WithBucket(bucketName)

  − .WithKey(path));

  −

  − if (!res.IsSuccess()) {

  − auto & error = res.GetError();

  − if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME

  − && error.GetMessage().find("404") != std::string::npos)

  − return false;

  − throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage());

  − }

  −

  − return true;

  − }

  −

  − void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string & data)

  − {

  − auto request =

  − Aws::S3::Model::PutObjectRequest()

  − .WithBucket(bucketName)

  − .WithKey(path);

  −

  − auto stream = std::make_shared<std::stringstream>(data);

  −

  − request.SetBody(stream);

  −

  − stats.put++;

  − stats.putBytes += data.size();

  −

  − auto now1 = std::chrono::steady_clock::now();

  −

  − auto result = checkAws(format("AWS error uploading ‘%s’") % path,

  − client->PutObject(request));

  −

  − auto now2 = std::chrono::steady_clock::now();

  −

  − auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

  −

  − printMsg(lvlInfo, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")

  − % bucketName % path % data.size() % duration);

  −

  − stats.putTimeMs += duration;

  − }

  −

  − std::shared_ptr<std::string> S3BinaryCacheStore::getFile(const std::string & path)

  − {

  − auto request =

  − Aws::S3::Model::GetObjectRequest()

  − .WithBucket(bucketName)

  − .WithKey(path);

  −

  − request.SetResponseStreamFactory([&]() {

  − return Aws::New<std::stringstream>("STRINGSTREAM");

  − });

  −

  − stats.get++;

  −

  − try {

  −

  − auto now1 = std::chrono::steady_clock::now();

  −

  − auto result = checkAws(format("AWS error fetching ‘%s’") % path,

  − client->GetObject(request));

  −

  − auto now2 = std::chrono::steady_clock::now();

  −

  − auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();

  −

  − auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

  −

  − printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")

  − % bucketName % path % res.size() % duration);

  −

  − stats.getBytes += res.size();

  − stats.getTimeMs += duration;

  −

  − return std::make_shared<std::string>(res);

  −

  − } catch (S3Error & e) {

  − if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return 0;

  − throw;

  − }

  − }

  −

  − }

  − printMsg(lvlDebug, format("fetching ‘s3://%1%/%2%’...") % bucketName % path);

  −

  −

  − struct S3Error : public Error

  − {

  − Aws::S3::S3Errors err;

  − S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)

  − : Error(fs), err(err) { };

  − };

  −

  − #include "nar-info.hh"

  − #include "nar-info-disk-cache.hh"
• file deletion: s3-binary-cache-store.hh (----------)
  [7.187]→[7.6356:6404](∅→∅),[7.6404]→[7.5519:5519](∅→∅)
  − #pragma once

  −

  − #include "binary-cache-store.hh"

  −

  − namespace Aws { namespace Client { class ClientConfiguration; } }

  − namespace Aws { namespace S3 { class S3Client; } }

  −

  − namespace nix {

  −

  − class S3BinaryCacheStore : public BinaryCacheStore

  − {

  − private:

  −

  − std::string bucketName;

  −

  − ref<Aws::Client::ClientConfiguration> config;

  − ref<Aws::S3::S3Client> client;

  −

  − public:

  −

  − S3BinaryCacheStore(std::shared_ptr<Store> localStore,

  − const Path & secretKeyFile, const std::string & bucketName);

  −

  − void init() override;

  −

  − private:

  −

  − ref<Aws::Client::ClientConfiguration> makeConfig();

  −

  − protected:

  −

  − bool fileExists(const std::string & path) override;

  −

  − void upsertFile(const std::string & path, const std::string & data) override;

  −

  − std::shared_ptr<std::string> getFile(const std::string & path) override;

  −

  − };

  −

  − }

  −

  − Stats stats;

  − struct Stats

  − {

  − std::atomic<uint64_t> put{0};

  − std::atomic<uint64_t> putBytes{0};

  − std::atomic<uint64_t> putTimeMs{0};

  − std::atomic<uint64_t> get{0};

  − std::atomic<uint64_t> getBytes{0};

  − std::atomic<uint64_t> getTimeMs{0};

  − std::atomic<uint64_t> head{0};

  − };

  −

  − const Stats & getS3Stats();

  −

  −

  − bool isValidPathUncached(const Path & storePath) override;

  −

  − std::string getUri();

  −

  − #include <atomic>
• edit in src/hydra-queue-runner/Makefile.am at line 6
  [7.61]→[2.0:53](∅→∅)
  − s3-binary-cache-store.hh s3-binary-cache-store.cc \