Add an S3-backed binary cache store

[?]

Feb 18, 2016, 3:18 PM

GTUZLZRHJ6GL5BNXOO3GA6Y3GFO7AXLIVPQHSG26LCF42KC2N7LQC

Dependencies

[2] UQYHPQ6U Run PostgreSQL during "make check"
[3] N2NKSKHS Refactor local binary cache code into a subclass
[4] 7VQ4ALFY Update "make check" for the new queue runner
[5] GH4S4AWM Rename file
[6] 24BMQDZA Start of single-process hydra-queue-runner
[7] N4IROACV Move buildRemote() into State
[8] W2AOTSS6 Rename class
[9] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
[10] 3FQ65IXO hydra-queue-runner: Compress binary cache NARs using xz
[11] HJOEIMLR Refactor
[12] 5AIYUMTB Basic remote building
[13] 32HHP5CW hydra-queue-runner: Support generating a signed binary cache
[*] T4LLYESZ * Nix expression for building Hydra.
[*] 6K5PBUUN Use buildEnv to combine Hydra's Perl dependencies
[*] ENXUSMSV Make concurrency more robust

Change contents

edit in release.nix at line 162

[2.47]

[16.1420]

(aws-sdk-cpp.override {
apis = ["s3"];
customMemoryManagement = false;
})
replacement in src/hydra-queue-runner/Makefile.am at line 7

[3.47]→[3.47:104](∅→∅)

local-binary-cache-store.hh local-binary-cache-store.cc

[3.47]

[4.322]

local-binary-cache-store.hh local-binary-cache-store.cc \
s3-binary-cache-store.hh s3-binary-cache-store.cc
replacement in src/hydra-queue-runner/Makefile.am at line 11

[4.369]→[4.369:403](∅→∅)

AM_CXXFLAGS = $(NIX_CFLAGS) -Wall

[4.369]

AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3
edit in src/hydra-queue-runner/binary-cache-store.cc at line 9

[4.2014]

[4.2014]

#include <chrono>
edit in src/hydra-queue-runner/binary-cache-store.cc at line 54

[4.3049]→[4.3049:3138](∅→∅),[4.3138]→[4.1118:1155](∅→∅)

printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
% info.path % info.narSize);
edit in src/hydra-queue-runner/binary-cache-store.cc at line 57

[4.1188]

[4.149]

auto now1 = std::chrono::steady_clock::now();
edit in src/hydra-queue-runner/binary-cache-store.cc at line 59

[4.185]

[4.1189]

auto now2 = std::chrono::steady_clock::now();
edit in src/hydra-queue-runner/binary-cache-store.cc at line 62

[4.1278]

[4.3170]

printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% info.path % info.narSize
% ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
% std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 11

[3.1356]

[17.19]

#include "s3-binary-cache-store.hh"
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 37

[4.1660]

[3.1357]

#if 0
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 39

[4.1001]→[4.55:84](∅→∅),[3.1423]→[4.55:84](∅→∅),[4.55]→[4.55:84](∅→∅)

"/tmp/binary-cache",

[3.1423]

[4.84]

"/home/eelco/Misc/Keys/test.nixos.org/secret",
"/home/eelco/Misc/Keys/test.nixos.org/public",
"/tmp/binary-cache");
#endif
auto store = make_ref<S3BinaryCacheStore>(getLocalStore(),
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 45

[4.139]→[4.139:195](∅→∅)

"/home/eelco/Misc/Keys/test.nixos.org/public");

[4.139]

[3.1424]

"/home/eelco/Misc/Keys/test.nixos.org/public",
"nix-test-cache-3");
replacement in src/hydra-queue-runner/local-binary-cache-store.cc at line 6

[3.1589]→[3.1589:1679](∅→∅)

const Path & binaryCacheDir, const Path & secretKeyFile, const Path & publicKeyFile)

[3.1589]

[3.1679]

const Path & secretKeyFile, const Path & publicKeyFile,
const Path & binaryCacheDir)
replacement in src/hydra-queue-runner/local-binary-cache-store.hh at line 15

[3.2839]→[3.2839:2982](∅→∅)

LocalBinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
const Path & secretKeyFile, const Path & publicKeyFile);

[3.2839]

[3.2982]

LocalBinaryCacheStore(ref<Store> localStore,
const Path & secretKeyFile, const Path & publicKeyFile,
const Path & binaryCacheDir);
file addition: s3-binary-cache-store.cc (----------)

[4.187]

#include "s3-binary-cache-store.hh"

#include <aws/core/client/ClientConfiguration.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/GetBucketLocationRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/PutObjectRequest.h>

namespace nix {

/* Helper: given an Outcome<R, E>, return R in case of success, or
throw an exception in case of an error. */
template<typename R, typename E>
R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
{
if (!outcome.IsSuccess())
throw Error(format("AWS error: %1%") % outcome.GetError().GetMessage());
return outcome.GetResultWithOwnership();
}

S3BinaryCacheStore::S3BinaryCacheStore(ref<Store> localStore,
const Path & secretKeyFile, const Path & publicKeyFile,
const std::string & bucketName)
: BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
, bucketName(bucketName)
, config(makeConfig())
, client(make_ref<Aws::S3::S3Client>(*config))
{
}

ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()
{
auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = Aws::Region::EU_WEST_1;
res->requestTimeoutMs = 600 * 1000;
return res;
}

void S3BinaryCacheStore::init()
{
/* Create the bucket if it doesn't already exists. */
// FIXME: HeadBucket would be more appropriate, but doesn't return
// an easily parsed 404 message.
auto res = client->GetBucketLocation(
Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

if (!res.IsSuccess()) {
if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
throw Error(format("AWS error: %1%") % res.GetError().GetMessage());

checkAws(client->CreateBucket(
Aws::S3::Model::CreateBucketRequest()
.WithBucket(bucketName)
.WithCreateBucketConfiguration(
Aws::S3::Model::CreateBucketConfiguration()
.WithLocationConstraint(
Aws::S3::Model::BucketLocationConstraint::eu_west_1))));
}

BinaryCacheStore::init();
}

bool S3BinaryCacheStore::fileExists(const std::string & path)
{
auto res = client->HeadObject(
Aws::S3::Model::HeadObjectRequest()
.WithBucket(bucketName)
.WithKey(path));

if (!res.IsSuccess()) {
auto & error = res.GetError();
if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
&& error.GetMessage().find("404") != std::string::npos)
return false;
throw Error(format("AWS error: %1%") % error.GetMessage());
}

return true;
}

void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
{
auto request =
Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);

auto stream = std::make_shared<std::stringstream>(data);

request.SetBody(stream);

auto now1 = std::chrono::steady_clock::now();

auto result = checkAws(client->PutObject(request));

auto now2 = std::chrono::steady_clock::now();

printMsg(lvlError, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
% bucketName % path
% data.size()
% std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
}

std::string S3BinaryCacheStore::getFile(const std::string & path)
{
auto request =
Aws::S3::Model::GetObjectRequest()
.WithBucket(bucketName)
.WithKey(path);

request.SetResponseStreamFactory([&]() {
return Aws::New<std::stringstream>("STRINGSTREAM");
});

auto now1 = std::chrono::steady_clock::now();

auto result = checkAws(client->GetObject(request));

auto now2 = std::chrono::steady_clock::now();

auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();

printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3%) in %4% ms")
% bucketName % path
% res.size()
% std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());

return res;
}

}
file addition: s3-binary-cache-store.hh (----------)

[4.187]

#pragma once

#include "binary-cache-store.hh"

namespace Aws { namespace Client { class ClientConfiguration; } }
namespace Aws { namespace S3 { class S3Client; } }

namespace nix {

class S3BinaryCacheStore : public BinaryCacheStore
{
private:

std::string bucketName;

ref<Aws::Client::ClientConfiguration> config;
ref<Aws::S3::S3Client> client;

public:

S3BinaryCacheStore(ref<Store> localStore,
const Path & secretKeyFile, const Path & publicKeyFile,
const std::string & bucketName);

void init() override;

private:

ref<Aws::Client::ClientConfiguration> makeConfig();

protected:

bool fileExists(const std::string & path) override;

void upsertFile(const std::string & path, const std::string & data) override;

std::string getFile(const std::string & path) override;

};

}