grahamc/hydra - Change N3G7LLGEYREJGGZFBFN4W3LPXQO4LPLTEATUFMUSNA3HXXH5KKKAC

Get data needed by getBuildOutput() from the incoming NAR in a streaming fashion

Created by Eelco Dolstra on July 27, 2020

N3G7LLGEYREJGGZFBFN4W3LPXQO4LPLTEATUFMUSNA3HXXH5KKKAC

Dependencies

In channels

main

upstream

Change contents

Replacement in src/hydra-queue-runner/Makefile.am at line 4 [5.219]

B:BD[4.84] → [4.84:131]

B:BD[4.131] → [2.0:43]

 builder.cc build-result.cc build-remote.cc  \
 build-result.hh counter.hh state.hh db.hh

[4.84]

[6.299]

 builder.cc build-result.cc build-remote.cc \
 build-result.hh counter.hh state.hh db.hh \
 nar-extractor.cc nar-extractor.hh

Replacement in src/hydra-queue-runner/build-remote.cc at line 163 [8.85]

B:BD[7.199] → [7.199:246]

    std::function<void(StepState)> updateStep)

[7.199]

[8.2925]

    std::function<void(StepState)> updateStep,
    NarMemberDatas & narMembers)

Deletion in src/hydra-queue-runner/build-remote.cc at line 431 [8.85]
B:BD[9.7000] → [10.559:614]
```
        result.accessor = destStore->getFSAccessor();
```

Replacement in src/hydra-queue-runner/build-remote.cc at line 477 [8.85]

B:BD[3.2000] → [3.2000:2051]

                destStore->addToStore(info, from);

[3.2000]

[3.2051]


                /* Receive the NAR from the remote and add it to the
                   destination store. Meanwhile, extract all the info from the
                   NAR that getBuildOutput() needs. */
                auto source2 = sinkToSource([&](Sink & sink)
                {
                    TeeSource tee(from, sink);
                    extractNarData(tee, localStore->printStorePath(path), narMembers);
                });
                destStore->addToStore(info, *source2);

Replacement in src/hydra-queue-runner/build-result.cc at line 11 [5.441]

B:BD[5.555] → [11.27:141]

BuildOutput getBuildOutput(nix::ref<Store> store,
    nix::ref<nix::FSAccessor> accessor, const Derivation & drv)

[5.555]

[5.606]

BuildOutput getBuildOutput(
    nix::ref<Store> store,
    NarMemberDatas & narMembers,
    const Derivation & drv)

Insertion in src/hydra-queue-runner/build-result.cc at line 27 [5.441]

[12.2930]

[5.1073]

    }
    /* Fetch missing data. Usually buildRemote() will have extracted
       this data from the incoming NARs. */
    for (auto & output : outputs) {
        auto outputS = store->printStorePath(output);
        if (!narMembers.count(outputS)) {
            printInfo("fetching NAR contents of '%s'...", outputS);
            auto source = sinkToSource([&](Sink & sink)
            {
                store->narFromPath(output, sink);
            });
            extractNarData(*source, outputS, narMembers);
        }

Replacement in src/hydra-queue-runner/build-result.cc at line 58 [5.441]

B:BD[13.3493] → [13.3493:3552]

∅:D[12.3045] → [11.142:217]

∅:D[13.3552] → [11.142:217]

B:BD[14.58] → [11.142:217]

        Path failedFile = outputS + "/nix-support/failed";
        if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)

[13.3493]

[11.217]

        if (narMembers.count(outputS + "/nix-support/failed"))

Replacement in src/hydra-queue-runner/build-result.cc at line 61 [5.441]

B:BD[14.114] → [13.3553:3628]

∅:D[12.3121] → [11.323:400]

∅:D[13.3628] → [11.323:400]

B:BD[11.323] → [11.323:400]

        Path productsFile = outputS + "/nix-support/hydra-build-products";
        if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)

[14.114]

[11.400]

        auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
        if (productsFile == narMembers.end() ||
            productsFile->second.type != FSAccessor::Type::tRegular)

Insertion in src/hydra-queue-runner/build-result.cc at line 65 [5.441]
[11.422]
[15.523]
```
        assert(productsFile->second.contents);
```

Replacement in src/hydra-queue-runner/build-result.cc at line 69 [5.441]

B:BD[5.1339] → [11.423:517]

        for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {

[5.1339]

[16.640]

        for (auto & line : tokenizeString<Strings>(productsFile->second.contents.value(), "\n")) {

Replacement in src/hydra-queue-runner/build-result.cc at line 89 [5.441]

B:BD[5.2528] → [11.760:877]

            auto st = accessor->stat(product.path);
            if (st.type == FSAccessor::Type::tMissing) continue;

[5.2528]

[5.2628]

            auto file = narMembers.find(product.path);
            if (file == narMembers.end()) continue;

Replacement in src/hydra-queue-runner/build-result.cc at line 94 [5.441]

B:BD[5.2877] → [11.878:935]

            if (st.type == FSAccessor::Type::tRegular) {

[5.2877]

[5.2916]

            if (file->second.type == FSAccessor::Type::tRegular) {

Replacement in src/hydra-queue-runner/build-result.cc at line 96 [5.441]

B:BD[5.2958] → [11.936:1050]

B:BD[11.1115] → [11.1115:1184]

                product.fileSize = st.fileSize;
                auto contents = accessor->readFile(product.path);
                product.sha256hash = hashString(htSHA256, contents);

[5.2958]

[5.3143]

                product.fileSize = file->second.fileSize.value();
                product.sha256hash = file->second.sha256.value();

Replacement in src/hydra-queue-runner/build-result.cc at line 114 [5.441]

B:BD[5.3665] → [11.1185:1292]

B:BD[11.1292] → [13.3862:3935]

∅:D[12.3428] → [11.1375:1432]

∅:D[13.3935] → [11.1375:1432]

B:BD[11.1375] → [11.1375:1432]

            auto st = accessor->stat(product.path);
            if (st.type == FSAccessor::Type::tMissing)
                throw Error("getting status of ‘%s’", product.path);
            if (st.type == FSAccessor::Type::tDirectory)

[5.3665]

[5.3865]

            auto file = narMembers.find(product.path);
            assert(file != narMembers.end());
            if (file->second.type == FSAccessor::Type::tDirectory)

Replacement in src/hydra-queue-runner/build-result.cc at line 123 [5.441]

B:BD[5.4043] → [13.3936:4020]

∅:D[12.3513] → [11.1433:1509]

∅:D[13.4020] → [11.1433:1509]

B:BD[5.4104] → [11.1433:1509]

∅:D[11.1509] → [16.1103:1117]

B:BD[5.4142] → [16.1103:1117]

B:BD[16.1117] → [11.1510:1569]

∅:D[11.1569] → [16.1166:1208]

B:BD[16.1166] → [16.1166:1208]

        auto p = store->printStorePath(output) + "/nix-support/hydra-release-name";
        if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
        try {
            res.releaseName = trim(accessor->readFile(p));
        } catch (Error & e) { continue; }

[5.4043]

[5.4222]

        auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
        if (file == narMembers.end() ||
            file->second.type != FSAccessor::Type::tRegular)
            continue;
        res.releaseName = trim(file->second.contents.value());

Replacement in src/hydra-queue-runner/build-result.cc at line 133 [5.441]

B:BD[15.664] → [13.4021:4110]

∅:D[12.3603] → [11.1636:1815]

∅:D[13.4110] → [11.1636:1815]

B:BD[11.1636] → [11.1636:1815]

        auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics";
        if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
        for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {

[15.664]

[15.814]

        auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
        if (file == narMembers.end() ||
            file->second.type != FSAccessor::Type::tRegular)
            continue;
        for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {

Insertion in src/hydra-queue-runner/build-result.hh at line 8 [5.4329]
[11.1840]
[5.4389]
```
#include "nar-extractor.hh"
```

Replacement in src/hydra-queue-runner/build-result.hh at line 42 [5.4329]

B:BD[5.4741] → [11.1841:1966]

BuildOutput getBuildOutput(nix::ref<nix::Store> store,
    nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);

[5.4741]

BuildOutput getBuildOutput(
    nix::ref<nix::Store> store,
    NarMemberDatas & narMembers,
    const nix::Derivation & drv);

Insertion in src/hydra-queue-runner/builder.cc at line 204 [4.133]
[4.4289]
[4.4289]
```
        NarMemberDatas narMembers;
```

Replacement in src/hydra-queue-runner/builder.cc at line 208 [4.133]

B:BD[4.4376] → [7.591:712]

            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep);

[4.4376]

[4.4474]

            buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers);

Replacement in src/hydra-queue-runner/builder.cc at line 223 [4.133]

B:BD[7.801] → [12.4580:4671]

            res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), *step->drv);

[7.801]

[7.802]

            res = getBuildOutput(destStore, narMembers, *step->drv);

Deletion in src/hydra-queue-runner/builder.cc at line 225 [4.133]
∅:D[7.812] → [17.669:699]
B:BD[18.441] → [17.669:699]
```
        result.accessor = 0;
```

File addition: nar-extractor.cc (----------)

[5.187]

#include "nar-extractor.hh"
#include "archive.hh"
#include <unordered_set>
using namespace nix;
struct Extractor : ParseSink
{
    std::unordered_set<Path> filesToKeep {
        "/nix-support/hydra-build-products",
        "/nix-support/hydra-release-name",
        "/nix-support/hydra-metrics",
    };
    NarMemberDatas & members;
    NarMemberData * curMember = nullptr;
    Path prefix;
    Extractor(NarMemberDatas & members, const Path & prefix)
        : members(members), prefix(prefix)
    { }
    void createDirectory(const Path & path) override
    {
        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
    }
    void createRegularFile(const Path & path) override
    {
        curMember = &members.insert_or_assign(prefix + path, NarMemberData {
            .type = FSAccessor::Type::tRegular,
            .fileSize = 0,
            .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
        }).first->second;
    }
    std::optional<unsigned long long> expectedSize;
    std::unique_ptr<HashSink> hashSink;
    void preallocateContents(unsigned long long size) override
    {
        expectedSize = size;
        hashSink = std::make_unique<HashSink>(htSHA256);
    }
    void receiveContents(unsigned char * data, unsigned int len) override
    {
        assert(expectedSize);
        assert(curMember);
        assert(hashSink);
        *curMember->fileSize += len;
        (*hashSink)(data, len);
        if (curMember->contents) {
            curMember->contents->append((char *) data, len);
        }
        assert(curMember->fileSize <= expectedSize);
        if (curMember->fileSize == expectedSize) {
            auto [hash, len] = hashSink->finish();
            assert(curMember->fileSize == len);
            curMember->sha256 = hash;
            hashSink.reset();
        }
    }
    void createSymlink(const Path & path, const string & target) override
    {
        members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
    }
};
void extractNarData(
    Source & source,
    const Path & prefix,
    NarMemberDatas & members)
{
    Extractor extractor(members, prefix);
    parseDump(extractor, source);
    // Note: this point may not be reached if we're in a coroutine.
}

File addition: nar-extractor.hh (----------)

[5.187]

#pragma once
#include "fs-accessor.hh"
#include "types.hh"
#include "serialise.hh"
#include "hash.hh"
struct NarMemberData
{
    nix::FSAccessor::Type type;
    std::optional<unsigned long long> fileSize;
    std::optional<std::string> contents;
    std::optional<nix::Hash> sha256;
};
typedef std::map<nix::Path, NarMemberData> NarMemberDatas;
/* Read a NAR from a source and get to some info about every file
   inside the NAR. */
void extractNarData(
    nix::Source & source,
    const nix::Path & prefix,
    NarMemberDatas & members);

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 675 [4.20067]

B:BD[19.2684] → [19.2684:2755]

    return getBuildOutput(destStore, destStore->getFSAccessor(), drv);

[19.2684]

[20.1123]

    NarMemberDatas narMembers;
    return getBuildOutput(destStore, narMembers, drv);

Insertion in src/hydra-queue-runner/state.hh at line 17 [22.1128]
[21.1567]
[22.1461]
```
#include "nar-extractor.hh"
```
Deletion in src/hydra-queue-runner/state.hh at line 68 [22.1128]
∅:D[17.780] → [18.442:489]
B:BD[23.1810] → [18.442:489]
```
    std::shared_ptr<nix::FSAccessor> accessor;
```

Replacement in src/hydra-queue-runner/state.hh at line 521 [22.1128]

B:BD[7.1809] → [7.1809:1861]

        std::function<void(StepState)> updateStep);

[7.1809]

[23.2016]

        std::function<void(StepState)> updateStep,
        NarMemberDatas & narMembers);