Keep better bytesReceived/bytesSent stats

[?]

Feb 26, 2016, 3:16 PM

DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC

Dependencies

[2] RSISSEU6 Enable substitution on the build machines
[3] UVNTWTWG Prevent download of NARs we just uploaded
[4] 6TY4LNHH Finish copyClosure
[5] FITVNQ2S Keep track of the time we spend copying to/from build machines
[6] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
[7] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
[8] 6EO3HVNA Merge remote-tracking branch 'origin/master' into binary-cache
[9] MB3TISH2 Rate-limit the number of threads copying closures at the same time
[10] DKJFD6JN Process Nix API changes
[11] DWFTK56E Keep track of how many threads are waiting
[12] 5AIYUMTB Basic remote building
[13] A2GL5FOZ Moar stats
[14] N4IROACV Move buildRemote() into State
[15] LE4VZIY5 More stats
[16] CNLNT3T4 Allow only 1 thread to send a closure to a given machine at the same time
[17] OCZ4LSGG Automatically retry aborted builds
[*] 24BMQDZA Start of single-process hydra-queue-runner

Change contents

edit in src/hydra-queue-runner/build-remote.cc at line 11

[4.298]

[4.298]

#include "finally.hh"
edit in src/hydra-queue-runner/build-remote.cc at line 79

[4.1406]→[4.0:25](∅→∅)

counter & bytesSent,
edit in src/hydra-queue-runner/build-remote.cc at line 105

[4.78]→[4.64:94](∅→∅),[4.94]→[4.280:338](∅→∅)

for (auto & p : missing)
bytesSent += destStore->queryPathInfo(p).narSize;
edit in src/hydra-queue-runner/build-remote.cc at line 115

[4.2443]→[4.388:438](∅→∅),[4.438]→[3.0:124](∅→∅),[3.124]→[4.2561:2563](∅→∅),[4.231]→[4.2561:2563](∅→∅),[4.2561]→[4.2561:2563](∅→∅),[4.2563]→[4.92:132](∅→∅),[4.132]→[4.2664:2680](∅→∅),[4.2664]→[4.2664:2680](∅→∅),[4.2680]→[3.125:176](∅→∅),[3.176]→[4.232:260](∅→∅),[4.480]→[4.232:260](∅→∅),[4.2717]→[4.232:260](∅→∅),[4.260]→[4.481:543](∅→∅),[4.318]→[4.2717:2721](∅→∅),[4.543]→[4.2717:2721](∅→∅),[4.2717]→[4.2717:2721](∅→∅)

static void copyClosureFrom(ref<Store> destStore,
FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived,
std::shared_ptr<FSAccessor> accessor)
{
to << cmdExportPaths << 0 << paths;
to.flush();
destStore->importPaths(false, from, accessor);

for (auto & p : paths)
bytesReceived += destStore->queryPathInfo(p).narSize;
}
edit in src/hydra-queue-runner/build-remote.cc at line 139

[4.3392]

[4.3392]

Finally updateStats([&]() {
bytesReceived += from.read;
bytesSent += to.written;
});
replacement in src/hydra-queue-runner/build-remote.cc at line 231

[4.56]→[2.0:69](∅→∅)

copyClosureTo(destStore, from, to, inputs, bytesSent, true);

[4.56]

[4.57]

copyClosureTo(destStore, from, to, inputs, true);
replacement in src/hydra-queue-runner/build-remote.cc at line 294

[4.273]→[3.233:320](∅→∅)

copyClosureFrom(destStore, from, to, outputs, bytesReceived, result.accessor);

[4.273]

[4.274]

to << cmdExportPaths << 0 << outputs;
to.flush();
destStore->importPaths(false, from, result.accessor);
file addition: finally.hh (----------)

[19.187]

#pragma once

/* A trivial class to run a function at the end of a scope. */
class Finally
{
private:
std::function<void()> fun;

public:
Finally(std::function<void()> fun) : fun(fun) { }
~Finally() { fun(); }
};