buildRemote(): Copy paths to the destination store in O(1) memory

[?]

Jul 27, 2020, 4:11 PM

5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC

Dependencies

[2] H46LUUZY hydra-queue-runner: don't try to distribute builds on localhost
[3] BTSZJ7W2 Quick hack to fix compilation
[4] 5CNQUQ77 Bump Nix
[5] EFWDY2V7 Support testing build determinism
[6] 5AIYUMTB Basic remote building
[7] 6EO3HVNA Merge remote-tracking branch 'origin/master' into binary-cache
[8] WV4SSAIY Build against nix-master
[9] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
[10] BUEWVH2M Remove signing parameter (nix#f435f82)
[11] FITVNQ2S Keep track of the time we spend copying to/from build machines
[12] EBJP3MNA Build against nix-master
[13] SL3WSRAC hydra-queue-runner: Limit memory usage
[14] LE4VZIY5 More stats
[15] PW4TLMWS Sync with latest Nix
[16] UVNTWTWG Prevent download of NARs we just uploaded
[*] 73YR46NJ hydra-queue-runner: Write directly to a binary cache

Change contents

edit in src/hydra-queue-runner/build-remote.cc at line 127

[4.2443]

[18.544]

// FIXME: use Store::topoSortPaths().
StorePaths topoSortPaths(const std::map<StorePath, ValidPathInfo> & paths)
{
StorePaths sorted;
StorePathSet visited;

std::function<void(const StorePath & path)> dfsVisit;

dfsVisit = [&](const StorePath & path) {
if (!visited.insert(path).second) return;

auto info = paths.find(path);
auto references = info == paths.end() ? StorePathSet() : info->second.references;

for (auto & i : references)
/* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */
if (i != path && paths.count(i))
dfsVisit(i);

sorted.push_back(path);
};

for (auto & i : paths)
dfsVisit(i.first);

std::reverse(sorted.begin(), sorted.end());

return sorted;
}
edit in src/hydra-queue-runner/build-remote.cc at line 183

[4.3316]

[4.11]

// FIXME: rewrite to use Store.
replacement in src/hydra-queue-runner/build-remote.cc at line 218

[4.415]→[4.0:42](∅→∅)

to << SERVE_MAGIC_1 << 0x203;

[4.415]

[4.457]

to << SERVE_MAGIC_1 << 0x204;
replacement in src/hydra-queue-runner/build-remote.cc at line 441

[4.47]→[4.7302:7356](∅→∅)

/* Query the size of the output paths. */

[4.47]

[4.7356]

/* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos;
replacement in src/hydra-queue-runner/build-remote.cc at line 448

[4.7492]→[4.7492:7543](∅→∅)

if (readString(from) == "") break;

[4.7492]

[4.7543]

auto storePathS = readString(from);
if (storePathS == "") break;
ValidPathInfo info(localStore->parseStorePath(storePathS));
assert(outputs.count(info.path));
replacement in src/hydra-queue-runner/build-remote.cc at line 453

[4.7588]→[4.7588:7646](∅→∅)

readStrings<PathSet>(from); // references

[4.7588]

[4.7646]

info.references = readStorePaths<StorePathSet>(*localStore, from);
replacement in src/hydra-queue-runner/build-remote.cc at line 455

[4.7699]→[4.7699:7751](∅→∅)

totalNarSize += readLongLong(from);

[4.7699]

[4.7751]

info.narSize = readLongLong(from);
totalNarSize += info.narSize;
info.narHash = Hash(readString(from), htSHA256);
info.ca = parseContentAddressOpt(readString(from));
readStrings<StringSet>(from); // sigs
infos.insert_or_assign(info.path, info);
edit in src/hydra-queue-runner/build-remote.cc at line 468

[4.104]

[4.2796]

/* Copy each path. */
replacement in src/hydra-queue-runner/build-remote.cc at line 472

[4.218]→[4.2648:2742](∅→∅),[4.2742]→[2.92:92](∅→∅),[2.92]→[4.8846:8870](∅→∅),[4.2742]→[4.8846:8870](∅→∅),[4.3249]→[4.8846:8870](∅→∅),[4.8846]→[4.8846:8870](∅→∅),[4.8870]→[3.0:78](∅→∅)

to << cmdExportPaths << 0;
writeStorePaths(*localStore, to, outputs);

to.flush();
destStore->importPaths(from, /* result.accessor, */ NoCheckSigs);

[4.218]

[4.231]

auto pathsSorted = topoSortPaths(infos);

for (auto & path : pathsSorted) {
auto & info = infos.find(path)->second;
to << cmdDumpStorePath << localStore->printStorePath(path);
to.flush();
destStore->addToStore(info, from);
}