grahamc/hydra2 - Change 2PQPKMG7PHKSWDYVLVHO5SJAWQT253NUOPKP6C65TXNUKOPMFQJQC

S3BinaryCacheStore: Use disk cache

Created by Eelco Dolstra on April 20, 2016

2PQPKMG7PHKSWDYVLVHO5SJAWQT253NUOPKP6C65TXNUKOPMFQJQC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/build-result.cc at line 24 [3.441]

B:BD[3.957] → [3.957:1073]

        res.closureSize += info.narSize;
        if (outputs.find(path) != outputs.end()) res.size += info.narSize;

[3.957]

[3.1073]

        res.closureSize += info->narSize;
        if (outputs.find(path) != outputs.end()) res.size += info->narSize;

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 632 [3.4840]
[4.2455]
[5.1781]
```
        auto store = getDestStore();
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 635 [3.4840]

B:BD[5.1782] → [5.1782:1857]

        auto store = dynamic_cast<S3BinaryCacheStore *>(&*getDestStore());

[5.1782]

[5.1857]

        root.attr("store");
        JSONObject nested(out);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 638 [3.4840]

B:BD[5.1858] → [5.1858:1947]

        if (store) {
            root.attr("store");
            JSONObject nested(out);

[5.1858]

[5.1947]

        auto & stats = store->getStats();
        nested.attr("narInfoRead", stats.narInfoRead);
        nested.attr("narInfoReadAverted", stats.narInfoReadAverted);
        nested.attr("narInfoMissing", stats.narInfoMissing);
        nested.attr("narInfoWrite", stats.narInfoWrite);
        nested.attr("narInfoCacheSize", stats.pathInfoCacheSize);
        nested.attr("narRead", stats.narRead);
        nested.attr("narReadBytes", stats.narReadBytes);
        nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
        nested.attr("narWrite", stats.narWrite);
        nested.attr("narWriteAverted", stats.narWriteAverted);
        nested.attr("narWriteBytes", stats.narWriteBytes);
        nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);
        nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);
        nested.attr("narCompressionSavings",
            stats.narWriteBytes
            ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
            : 0.0);
        nested.attr("narCompressionSpeed", // MiB/s
            stats.narWriteCompressionTimeMs
            ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
            : 0.0);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 661 [3.4840]

B:BD[5.1948] → [5.1948:2053]

B:BD[5.2053] → [6.844:917]

B:BD[6.917] → [2.0:65]

∅:D[2.65] → [5.2053:2114]

∅:D[6.917] → [5.2053:2114]

B:BD[5.2053] → [5.2053:2114]

B:BD[5.2114] → [6.918:987]

∅:D[6.987] → [5.2114:2830]

B:BD[5.2114] → [5.2114:2830]

            auto & stats = store->getStats();
            nested.attr("narInfoRead", stats.narInfoRead);
            nested.attr("narInfoReadAverted", stats.narInfoReadAverted);
            nested.attr("narInfoMissing", stats.narInfoMissing);
            nested.attr("narInfoWrite", stats.narInfoWrite);
            nested.attr("narInfoCacheSize", stats.narInfoCacheSize);
            nested.attr("narRead", stats.narRead);
            nested.attr("narReadBytes", stats.narReadBytes);
            nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
            nested.attr("narWrite", stats.narWrite);
            nested.attr("narWriteAverted", stats.narWriteAverted);
            nested.attr("narWriteBytes", stats.narWriteBytes);
            nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);
            nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);
            nested.attr("narCompressionSavings",
                stats.narWriteBytes
                ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes

[5.1948]

[5.2830]

        auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
        if (s3Store) {
            nested.attr("s3");
            JSONObject nested2(out);
            auto & s3Stats = s3Store->getS3Stats();
            nested2.attr("put", s3Stats.put);
            nested2.attr("putBytes", s3Stats.putBytes);
            nested2.attr("putTimeMs", s3Stats.putTimeMs);
            nested2.attr("putSpeed",
                s3Stats.putTimeMs
                ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 673 [3.4840]

B:BD[5.2854] → [5.2854:3068]

            nested.attr("narCompressionSpeed", // MiB/s
                stats.narWriteCompressionTimeMs
                ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)

[5.2854]

[5.3068]

            nested2.attr("get", s3Stats.get);
            nested2.attr("getBytes", s3Stats.getBytes);
            nested2.attr("getTimeMs", s3Stats.getTimeMs);
            nested2.attr("getSpeed",
                s3Stats.getTimeMs
                ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 680 [3.4840]

B:BD[5.3092] → [5.3092:4388]


            auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
            if (s3Store) {
                nested.attr("s3");
                JSONObject nested2(out);
                auto & s3Stats = s3Store->getS3Stats();
                nested2.attr("put", s3Stats.put);
                nested2.attr("putBytes", s3Stats.putBytes);
                nested2.attr("putTimeMs", s3Stats.putTimeMs);
                nested2.attr("putSpeed",
                    s3Stats.putTimeMs
                    ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
                    : 0.0);
                nested2.attr("get", s3Stats.get);
                nested2.attr("getBytes", s3Stats.getBytes);
                nested2.attr("getTimeMs", s3Stats.getTimeMs);
                nested2.attr("getSpeed",
                    s3Stats.getTimeMs
                    ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
                    : 0.0);
                nested2.attr("head", s3Stats.head);
                nested2.attr("costDollarApprox",
                    (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
                    + s3Stats.put / 1000.0 * 0.005 +
                    + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
            }

[5.3092]

[5.4388]

            nested2.attr("head", s3Stats.head);
            nested2.attr("costDollarApprox",
                (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
                + s3Stats.put / 1000.0 * 0.005 +
                + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);

Insertion in src/hydra-queue-runner/s3-binary-cache-store.cc at line 4 [8.1297]
[7.64]
[8.1334]
```
#include "nar-info-disk-cache.hh"
```
Insertion in src/hydra-queue-runner/s3-binary-cache-store.cc at line 41 [8.1297]
[8.2347]
[8.2347]
```
{
    diskCache = getNarInfoDiskCache();
}
std::string S3BinaryCacheStore::getUri()
```
Insertion in src/hydra-queue-runner/s3-binary-cache-store.cc at line 47 [8.1297]
[8.2349]
[8.2349]
```
    return "s3://" + bucketName;
```

Replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 60 [8.1297]

B:BD[8.2621] → [8.2621:2905]

    /* Create the bucket if it doesn't already exists. */
    // FIXME: HeadBucket would be more appropriate, but doesn't return
    // an easily parsed 404 message.
    auto res = client->GetBucketLocation(
        Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

[8.2621]

[8.2905]

    if (!diskCache->cacheExists(getUri())) {

Replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 62 [8.1297]

B:BD[8.2906] → [8.2906:3014]

B:BD[8.3014] → [9.141:259]

    if (!res.IsSuccess()) {
        if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
            throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());

[8.2906]

[8.3095]

        /* Create the bucket if it doesn't already exists. */
        // FIXME: HeadBucket would be more appropriate, but doesn't return
        // an easily parsed 404 message.
        auto res = client->GetBucketLocation(
            Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

Replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 68 [8.1297]

B:BD[8.3096] → [9.260:701]

        checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
            client->CreateBucket(
                Aws::S3::Model::CreateBucketRequest()
                .WithBucket(bucketName)
                .WithCreateBucketConfiguration(
                    Aws::S3::Model::CreateBucketConfiguration()
                    /* .WithLocationConstraint(
                       Aws::S3::Model::BucketLocationConstraint::US) */ )));

[8.3096]

[8.3443]

        if (!res.IsSuccess()) {
            if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
                throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
            checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
                client->CreateBucket(
                    Aws::S3::Model::CreateBucketRequest()
                    .WithBucket(bucketName)
                    .WithCreateBucketConfiguration(
                        Aws::S3::Model::CreateBucketConfiguration()
                        /* .WithLocationConstraint(
                           Aws::S3::Model::BucketLocationConstraint::US) */ )));
        }
        diskCache->createCache(getUri());

Replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 97 [8.1297]

B:BD[7.617] → [7.617:678]

bool S3BinaryCacheStore::isValidPath(const Path & storePath)

[7.617]

[7.678]

bool S3BinaryCacheStore::isValidPathUncached(const Path & storePath)

Replacement in src/hydra-queue-runner/s3-binary-cache-store.cc at line 100 [8.1297]
B:BD[7.690] → [7.690:722]
```
        readNarInfo(storePath);
```
[7.690]
[7.722]
```
        queryPathInfo(storePath);
```
Insertion in src/hydra-queue-runner/s3-binary-cache-store.hh at line 27 [8.5519]
[8.6068]
[8.6068]
```
    std::string getUri();
```

Replacement in src/hydra-queue-runner/s3-binary-cache-store.hh at line 43 [8.5519]

B:BD[7.861] → [7.861:916]

    bool isValidPath(const Path & storePath) override;

[7.861]

[5.5862]

    bool isValidPathUncached(const Path & storePath) override;