Get data needed by getBuildOutput() from the incoming NAR in a streaming fashion

[?]

Jul 27, 2020, 6:38 PM

N3G7LLGEYREJGGZFBFN4W3LPXQO4LPLTEATUFMUSNA3HXXH5KKKAC

Dependencies

[2] ZMICO7M6 Remove TokenServer in preparation of making NAR copying O(1) memory
[3] 5JMO43B4 buildRemote(): Copy paths to the destination store in O(1) memory
[4] EBJP3MNA Build against nix-master
[5] YNO7CQ6P hydra-queue-runner: More accurate memory accounting
[6] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
[7] BG6PEOB2 Make the output size limit configurable
[8] 24BMQDZA Start of single-process hydra-queue-runner
[9] BAFICF73 Support hydra-build-products on binary cache stores
[10] WHULPA6S Handle failure with output
[11] G7KWXSFM Distinguish build step states
[12] WV4SSAIY Build against nix-master
[13] DKJFD6JN Process Nix API changes
[14] N4IROACV Move buildRemote() into State
[15] HPXFXHFS Remove SHA-1 hash from BuildProducts
[16] V6H6BWMK Sync with Nix
[17] UVNTWTWG Prevent download of NARs we just uploaded
[18] FITVNQ2S Keep track of the time we spend copying to/from build machines
[19] 2GRQJZT6 hydra-queue-runner: Support running in a NixOS container
[20] LVQXQIYA Kill active build steps when builds are cancelled
[21] YSZQ3ORR Fix build
[22] MHVIT4JY Split hydra-queue-runner.cc more
[23] YHP5DSOO Improve parsing of hydra-build-products
[24] 5AIYUMTB Basic remote building
[25] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
[26] EPWEMRI2 Allow determinism checking for entire jobsets
[27] 3YSJ3LYK Remove finally.hh
[28] XCDTFZUY hydra-queue-runner: Fix build
[29] 2DNPZFPN Step cancellation: Don't use pthread_cancel()
[30] U55WNIDP Abort unsupported build steps
[31] WDGARQ76 Reuse build products / metrics stored in the database
[32] ACBS7C6Q hydra-queue-runner: Detect changes to the scheduling shares
[33] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
[34] T5BIOVJE Add support for tracking custom metrics
[35] NJJ7H64S Very basic multi-threaded queue runner
[*] HJOEIMLR Refactor
[*] SL3WSRAC hydra-queue-runner: Limit memory usage

Change contents

replacement in src/hydra-queue-runner/Makefile.am at line 4

[4.84]→[4.84:131](∅→∅),[4.131]→[2.0:43](∅→∅)

builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh

[4.84]

[4.299]

builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh state.hh db.hh \
nar-extractor.cc nar-extractor.hh
replacement in src/hydra-queue-runner/build-remote.cc at line 163

[4.199]→[4.199:246](∅→∅)

std::function<void(StepState)> updateStep)

[4.199]

[4.2925]

std::function<void(StepState)> updateStep,
NarMemberDatas & narMembers)
edit in src/hydra-queue-runner/build-remote.cc at line 431

[4.7000]→[4.559:614](∅→∅)

result.accessor = destStore->getFSAccessor();
replacement in src/hydra-queue-runner/build-remote.cc at line 477

[3.2000]→[3.2000:2051](∅→∅)

destStore->addToStore(info, from);

[3.2000]

[3.2051]

/* Receive the NAR from the remote and add it to the
destination store. Meanwhile, extract all the info from the
NAR that getBuildOutput() needs. */
auto source2 = sinkToSource([&](Sink & sink)
{
TeeSource tee(from, sink);
extractNarData(tee, localStore->printStorePath(path), narMembers);
});

destStore->addToStore(info, *source2);
replacement in src/hydra-queue-runner/build-result.cc at line 11

[4.555]→[4.27:141](∅→∅)

BuildOutput getBuildOutput(nix::ref<Store> store,
nix::ref<nix::FSAccessor> accessor, const Derivation & drv)

[4.555]

[4.606]

BuildOutput getBuildOutput(
nix::ref<Store> store,
NarMemberDatas & narMembers,
const Derivation & drv)
edit in src/hydra-queue-runner/build-result.cc at line 27

[4.2930]

[4.1073]

}

/* Fetch missing data. Usually buildRemote() will have extracted
this data from the incoming NARs. */
for (auto & output : outputs) {
auto outputS = store->printStorePath(output);
if (!narMembers.count(outputS)) {
printInfo("fetching NAR contents of '%s'...", outputS);
auto source = sinkToSource([&](Sink & sink)
{
store->narFromPath(output, sink);
});
extractNarData(*source, outputS, narMembers);
}
replacement in src/hydra-queue-runner/build-result.cc at line 58

[4.3493]→[4.3493:3552](∅→∅),[4.3045]→[4.142:217](∅→∅),[4.3552]→[4.142:217](∅→∅),[4.58]→[4.142:217](∅→∅)

Path failedFile = outputS + "/nix-support/failed";
if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)

[4.3493]

[4.217]

if (narMembers.count(outputS + "/nix-support/failed"))
replacement in src/hydra-queue-runner/build-result.cc at line 61

[4.114]→[4.3553:3628](∅→∅),[4.3121]→[4.323:400](∅→∅),[4.3628]→[4.323:400](∅→∅),[4.323]→[4.323:400](∅→∅)

Path productsFile = outputS + "/nix-support/hydra-build-products";
if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)

[4.114]

[4.400]

auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products");
if (productsFile == narMembers.end() ||
productsFile->second.type != FSAccessor::Type::tRegular)
edit in src/hydra-queue-runner/build-result.cc at line 65

[4.422]

[4.523]

assert(productsFile->second.contents);
replacement in src/hydra-queue-runner/build-result.cc at line 69

[4.1339]→[4.423:517](∅→∅)

for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {

[4.1339]

[4.640]

for (auto & line : tokenizeString<Strings>(productsFile->second.contents.value(), "\n")) {
replacement in src/hydra-queue-runner/build-result.cc at line 89

[4.2528]→[4.760:877](∅→∅)

auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing) continue;

[4.2528]

[4.2628]

auto file = narMembers.find(product.path);
if (file == narMembers.end()) continue;
replacement in src/hydra-queue-runner/build-result.cc at line 94

[4.2877]→[4.878:935](∅→∅)

if (st.type == FSAccessor::Type::tRegular) {

[4.2877]

[4.2916]

if (file->second.type == FSAccessor::Type::tRegular) {
replacement in src/hydra-queue-runner/build-result.cc at line 96

[4.2958]→[4.936:1050](∅→∅),[4.1115]→[4.1115:1184](∅→∅)

product.fileSize = st.fileSize;
auto contents = accessor->readFile(product.path);
product.sha256hash = hashString(htSHA256, contents);

[4.2958]

[4.3143]

product.fileSize = file->second.fileSize.value();
product.sha256hash = file->second.sha256.value();
replacement in src/hydra-queue-runner/build-result.cc at line 114

[4.3665]→[4.1185:1292](∅→∅),[4.1292]→[4.3862:3935](∅→∅),[4.3428]→[4.1375:1432](∅→∅),[4.3935]→[4.1375:1432](∅→∅),[4.1375]→[4.1375:1432](∅→∅)

auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing)
throw Error("getting status of ‘%s’", product.path);
if (st.type == FSAccessor::Type::tDirectory)

[4.3665]

[4.3865]

auto file = narMembers.find(product.path);
assert(file != narMembers.end());
if (file->second.type == FSAccessor::Type::tDirectory)
replacement in src/hydra-queue-runner/build-result.cc at line 123

[4.4043]→[4.3936:4020](∅→∅),[4.3513]→[4.1433:1509](∅→∅),[4.4020]→[4.1433:1509](∅→∅),[4.4104]→[4.1433:1509](∅→∅),[4.1509]→[4.1103:1117](∅→∅),[4.4142]→[4.1103:1117](∅→∅),[4.1117]→[4.1510:1569](∅→∅),[4.1569]→[4.1166:1208](∅→∅),[4.1166]→[4.1166:1208](∅→∅)

auto p = store->printStorePath(output) + "/nix-support/hydra-release-name";
if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
try {
res.releaseName = trim(accessor->readFile(p));
} catch (Error & e) { continue; }

[4.4043]

[4.4222]

auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
continue;
res.releaseName = trim(file->second.contents.value());
replacement in src/hydra-queue-runner/build-result.cc at line 133

[4.664]→[4.4021:4110](∅→∅),[4.3603]→[4.1636:1815](∅→∅),[4.4110]→[4.1636:1815](∅→∅),[4.1636]→[4.1636:1815](∅→∅)

auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics";
if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {

[4.664]

[4.814]

auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics");
if (file == narMembers.end() ||
file->second.type != FSAccessor::Type::tRegular)
continue;
for (auto & line : tokenizeString<Strings>(file->second.contents.value(), "\n")) {
edit in src/hydra-queue-runner/build-result.hh at line 8

[4.1840]

[4.4389]

#include "nar-extractor.hh"
replacement in src/hydra-queue-runner/build-result.hh at line 42

[4.4741]→[4.1841:1966](∅→∅)

BuildOutput getBuildOutput(nix::ref<nix::Store> store,
nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);

[4.4741]

BuildOutput getBuildOutput(
nix::ref<nix::Store> store,
NarMemberDatas & narMembers,
const nix::Derivation & drv);
edit in src/hydra-queue-runner/builder.cc at line 204

[4.4289]

[4.4289]

NarMemberDatas narMembers;
replacement in src/hydra-queue-runner/builder.cc at line 208

[4.4376]→[4.591:712](∅→∅)

buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep);

[4.4376]

[4.4474]

buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep, updateStep, narMembers);
replacement in src/hydra-queue-runner/builder.cc at line 223

[4.801]→[4.4580:4671](∅→∅)

res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), *step->drv);

[4.801]

[4.802]

res = getBuildOutput(destStore, narMembers, *step->drv);
edit in src/hydra-queue-runner/builder.cc at line 225

[4.812]→[4.669:699](∅→∅),[4.441]→[4.669:699](∅→∅)

result.accessor = 0;
file addition: nar-extractor.cc (----------)

[4.187]

#include "nar-extractor.hh"

#include "archive.hh"

#include <unordered_set>

using namespace nix;

struct Extractor : ParseSink
{
std::unordered_set<Path> filesToKeep {
"/nix-support/hydra-build-products",
"/nix-support/hydra-release-name",
"/nix-support/hydra-metrics",
};

NarMemberDatas & members;
NarMemberData * curMember = nullptr;
Path prefix;

Extractor(NarMemberDatas & members, const Path & prefix)
: members(members), prefix(prefix)
{ }

void createDirectory(const Path & path) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tDirectory });
}

void createRegularFile(const Path & path) override
{
curMember = &members.insert_or_assign(prefix + path, NarMemberData {
.type = FSAccessor::Type::tRegular,
.fileSize = 0,
.contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
}).first->second;
}

std::optional<unsigned long long> expectedSize;
std::unique_ptr<HashSink> hashSink;

void preallocateContents(unsigned long long size) override
{
expectedSize = size;
hashSink = std::make_unique<HashSink>(htSHA256);
}

void receiveContents(unsigned char * data, unsigned int len) override
{
assert(expectedSize);
assert(curMember);
assert(hashSink);
*curMember->fileSize += len;
(*hashSink)(data, len);
if (curMember->contents) {
curMember->contents->append((char *) data, len);
}
assert(curMember->fileSize <= expectedSize);
if (curMember->fileSize == expectedSize) {
auto [hash, len] = hashSink->finish();
assert(curMember->fileSize == len);
curMember->sha256 = hash;
hashSink.reset();
}
}

void createSymlink(const Path & path, const string & target) override
{
members.insert_or_assign(prefix + path, NarMemberData { .type = FSAccessor::Type::tSymlink });
}
};

void extractNarData(
Source & source,
const Path & prefix,
NarMemberDatas & members)
{
Extractor extractor(members, prefix);
parseDump(extractor, source);
// Note: this point may not be reached if we're in a coroutine.
}
file addition: nar-extractor.hh (----------)

[4.187]

#pragma once

#include "fs-accessor.hh"
#include "types.hh"
#include "serialise.hh"
#include "hash.hh"

struct NarMemberData
{
nix::FSAccessor::Type type;
std::optional<unsigned long long> fileSize;
std::optional<std::string> contents;
std::optional<nix::Hash> sha256;
};

typedef std::map<nix::Path, NarMemberData> NarMemberDatas;

/* Read a NAR from a source and get to some info about every file
inside the NAR. */
void extractNarData(
nix::Source & source,
const nix::Path & prefix,
NarMemberDatas & members);
replacement in src/hydra-queue-runner/queue-monitor.cc at line 675

[4.2684]→[4.2684:2755](∅→∅)

return getBuildOutput(destStore, destStore->getFSAccessor(), drv);

[4.2684]

[4.1123]

NarMemberDatas narMembers;
return getBuildOutput(destStore, narMembers, drv);
edit in src/hydra-queue-runner/state.hh at line 17

[38.1567]

[37.1461]

#include "nar-extractor.hh"
edit in src/hydra-queue-runner/state.hh at line 68

[4.780]→[4.442:489](∅→∅),[4.1810]→[4.442:489](∅→∅)

std::shared_ptr<nix::FSAccessor> accessor;
replacement in src/hydra-queue-runner/state.hh at line 521

[4.1809]→[4.1809:1861](∅→∅)

std::function<void(StepState)> updateStep);

[4.1809]

[4.2016]

std::function<void(StepState)> updateStep,
NarMemberDatas & narMembers);