{"header":{"message":"Add an S3-backed binary cache store","description":null,"timestamp":"2016-02-18T15:18:50Z","authors":[{"email":"eelco.dolstra@logicblox.com","name":"Eelco Dolstra"}]},"hash":"GTUZLZRHJ6GL5BNXOO3GA6Y3GFO7AXLIVPQHSG26LCF42KC2N7LQC","authors":[],"deps":{"hashes":[["GTUZLZRHJ6GL5BNXOO3GA6Y3GFO7AXLIVPQHSG26LCF42KC2N7LQC","Add an S3-backed binary cache store"],["AA",null],["UQYHPQ6UO7LIQ4P6NSCAEV677K52RQOQKCF4KSGNXMMFOQKY7IPQC","Run PostgreSQL during \"make check\""],["N2NKSKHSFR6SDMGZO2ALNDVIOLOEE26BPQHCSWUGEFE3DVVVV4MAC","Refactor local binary cache code into a subclass"],["GH4S4AWM2RUS3RUDU4E4TVQJF3N3LKM7OQQSZTG6H7NNMH6FKR7AC","Rename file"],["W2AOTSS6ZFJZBF3IZKWFXHFW5H4DNGLWCEDJTLMFWVAXL46SNM7QC","Rename class"],["7VQ4ALFYKJBFR46T3WZDMGOXNRR3QNJEJQVBYJM4HSJUOOUD6WBQC","Update \"make check\" for the new queue runner"],["3FQ65IXOIAWKEV67IXY7WRUX4OGMDYRA3RZJ2XPZ42AZXUANQ74QC","hydra-queue-runner: Compress binary cache NARs using xz"],["73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","hydra-queue-runner: Write directly to a binary cache"],["HJOEIMLRDVQ2KZI5HGL2HKGBM3AHP7YIKGKDAGFUNKRUXVRB24NAC","Refactor"],["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","Basic remote building"],["N4IROACVZ4MU73J5SM6WXJMKQSFR3VN5SOKENNNZNEGMTGB2Q3HAC","Move buildRemote() into State"],["24BMQDZAWDQ7VNIA7TIROXSOYLOJBNZ2E4264WHWNJAEN6ZB3UOAC","Start of single-process hydra-queue-runner"],["32HHP5CWAWJB2Q3Q6HYEC66YFGQAV6ESU7VEQ5ENBAQ5ZCRP4KGAC","hydra-queue-runner: Support generating a signed binary cache"],["T4LLYESZ2HUXSLKZ6GNBLVWUVG7R5IDFHYHYO773QIZ6QTOOXR2AC","* Nix expression for building Hydra."],["6K5PBUUN4GQAMOVX5BS6YYMHJ3PIF2PPZBTIEQ4R7BQNHC23GS3AC","Use buildEnv to combine Hydra's Perl dependencies"],["ENXUSMSVOU3AZFMH2ZXR4ZVPV2LRRQYQJ6IFX33YN6IH2ORSNSAAC","Make concurrency more robust"]],"deps":["UQYHPQ6UO7LIQ4P6NSCAEV677K52RQOQKCF4KSGNXMMFOQKY7IPQC","N2NKSKHSFR6SDMGZO2ALNDVIOLOEE26BPQHCSWUGEFE3DVVVV4MAC"],"known":["24BMQDZAWDQ7VNIA7TIROXSOYLOJBNZ2E4264WHWNJAEN6ZB3UOAC","5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","7VQ4ALFYKJBFR46T3WZDMGOXNRR3QNJEJQVBYJM4HSJUOOUD6WBQC","HJOEIMLRDVQ2KZI5HGL2HKGBM3AHP7YIKGKDAGFUNKRUXVRB24NAC","N4IROACVZ4MU73J5SM6WXJMKQSFR3VN5SOKENNNZNEGMTGB2Q3HAC","73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","3FQ65IXOIAWKEV67IXY7WRUX4OGMDYRA3RZJ2XPZ42AZXUANQ74QC","32HHP5CWAWJB2Q3Q6HYEC66YFGQAV6ESU7VEQ5ENBAQ5ZCRP4KGAC","GH4S4AWM2RUS3RUDU4E4TVQJF3N3LKM7OQQSZTG6H7NNMH6FKR7AC","W2AOTSS6ZFJZBF3IZKWFXHFW5H4DNGLWCEDJTLMFWVAXL46SNM7QC"]},"hunks":[{"Edit":{"inode":{"change":15,"pos":1},"path":"release.nix","line":162,"atom":{"NewVertex":{"up_context":[{"change":2,"pos":47}],"down_context":[{"change":16,"pos":1420}],"flag":"BLOCK","start":0,"end":118,"inode":{"change":15,"pos":1}}},"contents":[{"Add":" (aws-sdk-cpp.override {"},{"Add":" apis = [\"s3\"];"},{"Add":" customMemoryManagement = false;"},{"Add":" })"}]}},{"Replacement":{"inode":{"change":4,"pos":219},"path":"src/hydra-queue-runner/Makefile.am","line":7,"ins":{"up_context":[{"change":3,"pos":47}],"down_context":[{"change":4,"pos":322}],"flag":"BLOCK","start":119,"end":229,"inode":{"change":4,"pos":219}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":47},"to":{"change":3,"start":47,"end":104},"introduced_by":3}],"inode":{"change":4,"pos":219}},"ins_contents":[{"Add":" local-binary-cache-store.hh local-binary-cache-store.cc \\"},{"Add":" s3-binary-cache-store.hh s3-binary-cache-store.cc"}],"del_contents":[{"Del":" local-binary-cache-store.hh local-binary-cache-store.cc\n"}]}},{"Replacement":{"inode":{"change":4,"pos":219},"path":"src/hydra-queue-runner/Makefile.am","line":11,"ins":{"up_context":[{"change":4,"pos":369}],"down_context":[],"flag":"BLOCK","start":230,"end":281,"inode":{"change":4,"pos":219}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":369},"to":{"change":4,"start":369,"end":403},"introduced_by":4}],"inode":{"change":4,"pos":219}},"ins_contents":[{"Add":"AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3"}],"del_contents":[{"Del":"AM_CXXFLAGS = $(NIX_CFLAGS) -Wall\n"}]}},{"Edit":{"inode":{"change":4,"pos":1879},"path":"src/hydra-queue-runner/binary-cache-store.cc","line":9,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":2014}],"down_context":[{"change":4,"pos":2014}],"flag":"BLOCK","start":282,"end":301,"inode":{"change":4,"pos":1879}}},"contents":[{"Add":""},{"Add":"#include "}]}},{"Edit":{"inode":{"change":4,"pos":1879},"path":"src/hydra-queue-runner/binary-cache-store.cc","line":54,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":3049},"to":{"change":4,"start":3049,"end":3138},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":3138},"to":{"change":4,"start":1118,"end":1155},"introduced_by":4}],"inode":{"change":4,"pos":1879}}},"contents":[{"Del":"\n printMsg(lvlTalkative, format(\"copying path ‘%1%’ (%2% bytes) to binary cache\")\n"},{"Del":" % info.path % info.narSize);\n"}]}},{"Edit":{"inode":{"change":4,"pos":1879},"path":"src/hydra-queue-runner/binary-cache-store.cc","line":57,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":1188}],"down_context":[{"change":4,"pos":149}],"flag":"BLOCK","start":302,"end":352,"inode":{"change":4,"pos":1879}}},"contents":[{"Add":" auto now1 = std::chrono::steady_clock::now();"}]}},{"Edit":{"inode":{"change":4,"pos":1879},"path":"src/hydra-queue-runner/binary-cache-store.cc","line":59,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":185}],"down_context":[{"change":4,"pos":1189}],"flag":"BLOCK","start":353,"end":403,"inode":{"change":4,"pos":1879}}},"contents":[{"Add":" auto now2 = std::chrono::steady_clock::now();"}]}},{"Edit":{"inode":{"change":4,"pos":1879},"path":"src/hydra-queue-runner/binary-cache-store.cc","line":62,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":1278}],"down_context":[{"change":4,"pos":3170}],"flag":"BLOCK","start":404,"end":709,"inode":{"change":4,"pos":1879}}},"contents":[{"Add":""},{"Add":" printMsg(lvlTalkative, format(\"copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache\")"},{"Add":" % info.path % info.narSize"},{"Add":" % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)"},{"Add":" % std::chrono::duration_cast(now2 - now1).count());"}]}},{"Edit":{"inode":{"change":4,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":11,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":1356}],"down_context":[{"change":17,"pos":19}],"flag":"BLOCK","start":710,"end":746,"inode":{"change":4,"pos":4840}}},"contents":[{"Add":"#include \"s3-binary-cache-store.hh\""}]}},{"Edit":{"inode":{"change":4,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":37,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":1660}],"down_context":[{"change":3,"pos":1357}],"flag":"BLOCK","start":747,"end":753,"inode":{"change":4,"pos":4840}}},"contents":[{"Add":"#if 0"}]}},{"Replacement":{"inode":{"change":4,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":39,"ins":{"up_context":[{"change":3,"pos":1423}],"down_context":[{"change":4,"pos":84}],"flag":"BLOCK","start":754,"end":964,"inode":{"change":4,"pos":4840}},"del":{"edges":[{"previous":"","flag":"DELETED","from":{"change":4,"pos":1001},"to":{"change":4,"start":55,"end":84},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":3,"pos":1423},"to":{"change":4,"start":55,"end":84},"introduced_by":3},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":55},"to":{"change":4,"start":55,"end":84},"introduced_by":4}],"inode":{"change":4,"pos":4840}},"ins_contents":[{"Add":" \"/home/eelco/Misc/Keys/test.nixos.org/secret\","},{"Add":" \"/home/eelco/Misc/Keys/test.nixos.org/public\","},{"Add":" \"/tmp/binary-cache\");"},{"Add":"#endif"},{"Add":" auto store = make_ref(getLocalStore(),"}],"del_contents":[{"Del":" \"/tmp/binary-cache\",\n"}]}},{"Replacement":{"inode":{"change":4,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":45,"ins":{"up_context":[{"change":4,"pos":139}],"down_context":[{"change":3,"pos":1424}],"flag":"BLOCK","start":965,"end":1049,"inode":{"change":4,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":139},"to":{"change":4,"start":139,"end":195},"introduced_by":4}],"inode":{"change":4,"pos":4840}},"ins_contents":[{"Add":" \"/home/eelco/Misc/Keys/test.nixos.org/public\","},{"Add":" \"nix-test-cache-3\");"}],"del_contents":[{"Del":" \"/home/eelco/Misc/Keys/test.nixos.org/public\");\n"}]}},{"Replacement":{"inode":{"change":3,"pos":1463},"path":"src/hydra-queue-runner/local-binary-cache-store.cc","line":6,"ins":{"up_context":[{"change":3,"pos":1589}],"down_context":[{"change":3,"pos":1679}],"flag":"BLOCK","start":1050,"end":1143,"inode":{"change":3,"pos":1463}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":1589},"to":{"change":3,"start":1589,"end":1679},"introduced_by":3}],"inode":{"change":3,"pos":1463}},"ins_contents":[{"Add":" const Path & secretKeyFile, const Path & publicKeyFile,"},{"Add":" const Path & binaryCacheDir)"}],"del_contents":[{"Del":" const Path & binaryCacheDir, const Path & secretKeyFile, const Path & publicKeyFile)\n"}]}},{"Replacement":{"inode":{"change":3,"pos":2672},"path":"src/hydra-queue-runner/local-binary-cache-store.hh","line":15,"ins":{"up_context":[{"change":3,"pos":2839}],"down_context":[{"change":3,"pos":2982}],"flag":"BLOCK","start":1144,"end":1295,"inode":{"change":3,"pos":2672}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":2839},"to":{"change":3,"start":2839,"end":2982},"introduced_by":3}],"inode":{"change":3,"pos":2672}},"ins_contents":[{"Add":" LocalBinaryCacheStore(ref localStore,"},{"Add":" const Path & secretKeyFile, const Path & publicKeyFile,"},{"Add":" const Path & binaryCacheDir);"}],"del_contents":[{"Del":" LocalBinaryCacheStore(ref localStore, const Path & binaryCacheDir,\n const Path & secretKeyFile, const Path & publicKeyFile);\n"}]}},{"FileAdd":{"meta":{"basename":"s3-binary-cache-store.cc","metadata":0},"context":[{"change":4,"pos":187}],"contents":[{"Add":"#include \"s3-binary-cache-store.hh\""},{"Add":""},{"Add":"#include "},{"Add":"#include "},{"Add":"#include "},{"Add":"#include "},{"Add":"#include "},{"Add":"#include "},{"Add":"#include "},{"Add":""},{"Add":"namespace nix {"},{"Add":""},{"Add":"/* Helper: given an Outcome, return R in case of success, or"},{"Add":" throw an exception in case of an error. */"},{"Add":"template"},{"Add":"R && checkAws(Aws::Utils::Outcome && outcome)"},{"Add":"{"},{"Add":" if (!outcome.IsSuccess())"},{"Add":" throw Error(format(\"AWS error: %1%\") % outcome.GetError().GetMessage());"},{"Add":" return outcome.GetResultWithOwnership();"},{"Add":"}"},{"Add":""},{"Add":"S3BinaryCacheStore::S3BinaryCacheStore(ref localStore,"},{"Add":" const Path & secretKeyFile, const Path & publicKeyFile,"},{"Add":" const std::string & bucketName)"},{"Add":" : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)"},{"Add":" , bucketName(bucketName)"},{"Add":" , config(makeConfig())"},{"Add":" , client(make_ref(*config))"},{"Add":"{"},{"Add":"}"},{"Add":""},{"Add":"ref S3BinaryCacheStore::makeConfig()"},{"Add":"{"},{"Add":" auto res = make_ref();"},{"Add":" res->region = Aws::Region::EU_WEST_1;"},{"Add":" res->requestTimeoutMs = 600 * 1000;"},{"Add":" return res;"},{"Add":"}"},{"Add":""},{"Add":"void S3BinaryCacheStore::init()"},{"Add":"{"},{"Add":" /* Create the bucket if it doesn't already exists. */"},{"Add":" // FIXME: HeadBucket would be more appropriate, but doesn't return"},{"Add":" // an easily parsed 404 message."},{"Add":" auto res = client->GetBucketLocation("},{"Add":" Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));"},{"Add":""},{"Add":" if (!res.IsSuccess()) {"},{"Add":" if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)"},{"Add":" throw Error(format(\"AWS error: %1%\") % res.GetError().GetMessage());"},{"Add":""},{"Add":" checkAws(client->CreateBucket("},{"Add":" Aws::S3::Model::CreateBucketRequest()"},{"Add":" .WithBucket(bucketName)"},{"Add":" .WithCreateBucketConfiguration("},{"Add":" Aws::S3::Model::CreateBucketConfiguration()"},{"Add":" .WithLocationConstraint("},{"Add":" Aws::S3::Model::BucketLocationConstraint::eu_west_1))));"},{"Add":" }"},{"Add":""},{"Add":" BinaryCacheStore::init();"},{"Add":"}"},{"Add":""},{"Add":"bool S3BinaryCacheStore::fileExists(const std::string & path)"},{"Add":"{"},{"Add":" auto res = client->HeadObject("},{"Add":" Aws::S3::Model::HeadObjectRequest()"},{"Add":" .WithBucket(bucketName)"},{"Add":" .WithKey(path));"},{"Add":""},{"Add":" if (!res.IsSuccess()) {"},{"Add":" auto & error = res.GetError();"},{"Add":" if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME"},{"Add":" && error.GetMessage().find(\"404\") != std::string::npos)"},{"Add":" return false;"},{"Add":" throw Error(format(\"AWS error: %1%\") % error.GetMessage());"},{"Add":" }"},{"Add":""},{"Add":" return true;"},{"Add":"}"},{"Add":""},{"Add":"void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string & data)"},{"Add":"{"},{"Add":" auto request ="},{"Add":" Aws::S3::Model::PutObjectRequest()"},{"Add":" .WithBucket(bucketName)"},{"Add":" .WithKey(path);"},{"Add":""},{"Add":" auto stream = std::make_shared(data);"},{"Add":""},{"Add":" request.SetBody(stream);"},{"Add":""},{"Add":" auto now1 = std::chrono::steady_clock::now();"},{"Add":""},{"Add":" auto result = checkAws(client->PutObject(request));"},{"Add":""},{"Add":" auto now2 = std::chrono::steady_clock::now();"},{"Add":""},{"Add":" printMsg(lvlError, format(\"uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms\")"},{"Add":" % bucketName % path"},{"Add":" % data.size()"},{"Add":" % std::chrono::duration_cast(now2 - now1).count());"},{"Add":"}"},{"Add":""},{"Add":"std::string S3BinaryCacheStore::getFile(const std::string & path)"},{"Add":"{"},{"Add":" auto request ="},{"Add":" Aws::S3::Model::GetObjectRequest()"},{"Add":" .WithBucket(bucketName)"},{"Add":" .WithKey(path);"},{"Add":""},{"Add":" request.SetResponseStreamFactory([&]() {"},{"Add":" return Aws::New(\"STRINGSTREAM\");"},{"Add":" });"},{"Add":""},{"Add":" auto now1 = std::chrono::steady_clock::now();"},{"Add":""},{"Add":" auto result = checkAws(client->GetObject(request));"},{"Add":""},{"Add":" auto now2 = std::chrono::steady_clock::now();"},{"Add":""},{"Add":" auto res = dynamic_cast(result.GetBody()).str();"},{"Add":""},{"Add":" printMsg(lvlError, format(\"downloaded ‘s3://%1%/%2%’ (%3%) in %4% ms\")"},{"Add":" % bucketName % path"},{"Add":" % res.size()"},{"Add":" % std::chrono::duration_cast(now2 - now1).count());"},{"Add":""},{"Add":" return res;"},{"Add":"}"},{"Add":""},{"Add":"}"},{"Add":""}]}},{"FileAdd":{"meta":{"basename":"s3-binary-cache-store.hh","metadata":0},"context":[{"change":4,"pos":187}],"contents":[{"Add":"#pragma once"},{"Add":""},{"Add":"#include \"binary-cache-store.hh\""},{"Add":""},{"Add":"namespace Aws { namespace Client { class ClientConfiguration; } }"},{"Add":"namespace Aws { namespace S3 { class S3Client; } }"},{"Add":""},{"Add":"namespace nix {"},{"Add":""},{"Add":"class S3BinaryCacheStore : public BinaryCacheStore"},{"Add":"{"},{"Add":"private:"},{"Add":""},{"Add":" std::string bucketName;"},{"Add":""},{"Add":" ref config;"},{"Add":" ref client;"},{"Add":""},{"Add":"public:"},{"Add":""},{"Add":" S3BinaryCacheStore(ref localStore,"},{"Add":" const Path & secretKeyFile, const Path & publicKeyFile,"},{"Add":" const std::string & bucketName);"},{"Add":""},{"Add":" void init() override;"},{"Add":""},{"Add":"private:"},{"Add":""},{"Add":" ref makeConfig();"},{"Add":""},{"Add":"protected:"},{"Add":""},{"Add":" bool fileExists(const std::string & path) override;"},{"Add":""},{"Add":" void upsertFile(const std::string & path, const std::string & data) override;"},{"Add":""},{"Add":" std::string getFile(const std::string & path) override;"},{"Add":""},{"Add":"};"},{"Add":""},{"Add":"}"}]}}],"owner":"grahamc","repo":"hydra2","token":"gioHNzObxObJagpx7+43cV11CyObANueTE9MM9Uo/AY=","login":null}

…

Dependencies

Change contents