{"header":{"message":"Keep better bytesReceived/bytesSent stats","description":null,"timestamp":"2016-02-26T15:16:36Z","authors":[{"email":"eelco.dolstra@logicblox.com","name":"Eelco Dolstra"}]},"hash":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC","authors":[],"deps":{"hashes":[["DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC","Keep better bytesReceived/bytesSent stats"],["AA",null],["RSISSEU6WXFBDCN75A5Z3OIPC5TZFEROBU4K5MGV76BMXNKPFAYQC","Enable substitution on the build machines"],["UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC","Prevent download of NARs we just uploaded"],["OCZ4LSGGSCMSLGC3C32D5JUYYHS5CIPOKOAMADEFAFZOFXJ3YY3AC","Automatically retry aborted builds"],["N4IROACVZ4MU73J5SM6WXJMKQSFR3VN5SOKENNNZNEGMTGB2Q3HAC","Move buildRemote() into State"],["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","Basic remote building"],["73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","hydra-queue-runner: Write directly to a binary cache"],["6TY4LNHHAUSZNFDN34CHUWMZSQN5I3LNOOSV457M44FDGZB76X4AC","Finish copyClosure"],["LE4VZIY5VZ52FOP5QQRIJINWIMWTAPRTZTGO77JXUEPGRPRSQYMAC","More stats"],["NAYQT2GTCJPBFRSK7CBFX655F2NGTBPICJSCYG2CSCQ5NRDHZG6QC","hydra-queue-runner: Use cmdBuildDerivation"],["DKJFD6JNNK5LJMRGQABMJZKMFZLGY3ADKJWF6J4BBHEUPY3NB67QC","Process Nix API changes"],["MB3TISH2KYBIGY6XJKMN4HO2S6TCN2GORJENMECCKLXGGIRS2O2AC","Rate-limit the number of threads copying closures at the same time"],["A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC","Moar stats"],["DWFTK56EAGNPSKWWWY5MFPATQWO5PZGHF5D53JOEJ2DG5NLLYRBQC","Keep track of how many threads are waiting"],["CNLNT3T4IXU42GAC64CU5CEDTDLM2SHSJZEVHTMOCWYFXSQNC76QC","Allow only 1 thread to send a closure to a given machine at the same time"],["FITVNQ2SVM6KSOF5P3HHWJYQ3WMQYDJGAONCBIZ7OF7CPXGMA36QC","Keep track of the time we spend copying to/from build machines"],["6EO3HVNAAVJLBYTBSHXQADDTYF7JJU3ANHS5SKHSSROOW4AXIZ4AC","Merge remote-tracking branch 'origin/master' into binary-cache"],["24BMQDZAWDQ7VNIA7TIROXSOYLOJBNZ2E4264WHWNJAEN6ZB3UOAC","Start of single-process hydra-queue-runner"]],"deps":["RSISSEU6WXFBDCN75A5Z3OIPC5TZFEROBU4K5MGV76BMXNKPFAYQC","UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC"],"known":["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","6TY4LNHHAUSZNFDN34CHUWMZSQN5I3LNOOSV457M44FDGZB76X4AC","OCZ4LSGGSCMSLGC3C32D5JUYYHS5CIPOKOAMADEFAFZOFXJ3YY3AC","MB3TISH2KYBIGY6XJKMN4HO2S6TCN2GORJENMECCKLXGGIRS2O2AC","LE4VZIY5VZ52FOP5QQRIJINWIMWTAPRTZTGO77JXUEPGRPRSQYMAC","A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC","N4IROACVZ4MU73J5SM6WXJMKQSFR3VN5SOKENNNZNEGMTGB2Q3HAC","CNLNT3T4IXU42GAC64CU5CEDTDLM2SHSJZEVHTMOCWYFXSQNC76QC","DWFTK56EAGNPSKWWWY5MFPATQWO5PZGHF5D53JOEJ2DG5NLLYRBQC","NAYQT2GTCJPBFRSK7CBFX655F2NGTBPICJSCYG2CSCQ5NRDHZG6QC","DKJFD6JNNK5LJMRGQABMJZKMFZLGY3ADKJWF6J4BBHEUPY3NB67QC","73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","FITVNQ2SVM6KSOF5P3HHWJYQ3WMQYDJGAONCBIZ7OF7CPXGMA36QC","6EO3HVNAAVJLBYTBSHXQADDTYF7JJU3ANHS5SKHSSROOW4AXIZ4AC"]},"hunks":[{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":11,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":298}],"down_context":[{"change":4,"pos":298}],"flag":"BLOCK","start":0,"end":22,"inode":{"change":4,"pos":85}}},"contents":[{"Add":{"text":"#include \"finally.hh\"","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:21"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":79,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":1406},"to":{"change":4,"start":0,"end":25},"introduced_by":4}],"inode":{"change":4,"pos":85}}},"contents":[{"Del":{"text":" counter & bytesSent,","pos":"CNLNT3T4IXU42GAC64CU5CEDTDLM2SHSJZEVHTMOCWYFXSQNC76QC:24"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":105,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":78},"to":{"change":4,"start":64,"end":94},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":94},"to":{"change":4,"start":280,"end":338},"introduced_by":4}],"inode":{"change":4,"pos":85}}},"contents":[{"Del":{"text":"","pos":"A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC:64"}},{"Del":{"text":" for (auto & p : missing)","pos":"A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC:93"}},{"Del":{"text":" bytesSent += destStore->queryPathInfo(p).narSize;","pos":"73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC:337"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":115,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2443},"to":{"change":4,"start":388,"end":438},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":438},"to":{"change":3,"start":0,"end":124},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":3,"pos":124},"to":{"change":4,"start":2561,"end":2563},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":4,"pos":231},"to":{"change":4,"start":2561,"end":2563},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2561},"to":{"change":4,"start":2561,"end":2563},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2563},"to":{"change":4,"start":92,"end":132},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":132},"to":{"change":4,"start":2664,"end":2680},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2664},"to":{"change":4,"start":2664,"end":2680},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2680},"to":{"change":3,"start":125,"end":176},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":3,"pos":176},"to":{"change":4,"start":232,"end":260},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":4,"pos":480},"to":{"change":4,"start":232,"end":260},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2717},"to":{"change":4,"start":232,"end":260},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":260},"to":{"change":4,"start":481,"end":543},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":318},"to":{"change":4,"start":2717,"end":2721},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":543},"to":{"change":4,"start":2717,"end":2721},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2717},"to":{"change":4,"start":2717,"end":2721},"introduced_by":4}],"inode":{"change":4,"pos":85}}},"contents":[{"Del":{"text":"static void copyClosureFrom(ref destStore,","pos":"73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC:437"}},{"Del":{"text":" FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived,","pos":"UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC:81"}},{"Del":{"text":" std::shared_ptr accessor)","pos":"UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC:123"}},{"Del":{"text":"{","pos":"5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC:2562"}},{"Del":{"text":" to << cmdExportPaths << 0 << paths;","pos":"NAYQT2GTCJPBFRSK7CBFX655F2NGTBPICJSCYG2CSCQ5NRDHZG6QC:131"}},{"Del":{"text":" to.flush();","pos":"5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC:2679"}},{"Del":{"text":" destStore->importPaths(false, from, accessor);","pos":"UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC:175"}},{"Del":{"text":"","pos":"A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC:232"}},{"Del":{"text":" for (auto & p : paths)","pos":"A2GL5FOZ3UJ2NM5RPRWTNPFTKLBA54B2UC6UIYO4M3N3RFNC4BTAC:259"}},{"Del":{"text":" bytesReceived += destStore->queryPathInfo(p).narSize;","pos":"73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC:542"}},{"Del":{"text":"}","pos":"5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC:2718"}},{"Del":{"text":"","pos":"5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC:2719"}},{"Del":{"text":"","pos":"5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC:2720"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":139,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":3392}],"down_context":[{"change":4,"pos":3392}],"flag":"BLOCK","start":23,"end":133,"inode":{"change":4,"pos":85}}},"contents":[{"Add":{"text":"","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:23"}},{"Add":{"text":" Finally updateStats([&]() {","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:55"}},{"Add":{"text":" bytesReceived += from.read;","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:91"}},{"Add":{"text":" bytesSent += to.written;","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:124"}},{"Add":{"text":" });","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:132"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":231,"ins":{"up_context":[{"change":4,"pos":56}],"down_context":[{"change":4,"pos":57}],"flag":"BLOCK","start":134,"end":192,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":56},"to":{"change":2,"start":0,"end":69},"introduced_by":2}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" copyClosureTo(destStore, from, to, inputs, true);","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:191"}}],"del_contents":[{"Del":{"text":" copyClosureTo(destStore, from, to, inputs, bytesSent, true);","pos":"RSISSEU6WXFBDCN75A5Z3OIPC5TZFEROBU4K5MGV76BMXNKPFAYQC:68"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":294,"ins":{"up_context":[{"change":4,"pos":273}],"down_context":[{"change":4,"pos":274}],"flag":"BLOCK","start":193,"end":321,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":273},"to":{"change":3,"start":233,"end":320},"introduced_by":3}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" to << cmdExportPaths << 0 << outputs;","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:238"}},{"Add":{"text":" to.flush();","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:258"}},{"Add":{"text":" destStore->importPaths(false, from, result.accessor);","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:320"}}],"del_contents":[{"Del":{"text":" copyClosureFrom(destStore, from, to, outputs, bytesReceived, result.accessor);","pos":"UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC:319"}}]}},{"FileAdd":{"meta":{"basename":"finally.hh","metadata":0},"context":[{"change":19,"pos":187}],"contents":[{"Add":{"text":"#pragma once","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:336"}},{"Add":{"text":"","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:337"}},{"Add":{"text":"/* A trivial class to run a function at the end of a scope. */","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:400"}},{"Add":{"text":"class Finally","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:414"}},{"Add":{"text":"{","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:416"}},{"Add":{"text":"private:","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:425"}},{"Add":{"text":" std::function fun;","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:456"}},{"Add":{"text":"","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:457"}},{"Add":{"text":"public:","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:465"}},{"Add":{"text":" Finally(std::function fun) : fun(fun) { }","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:519"}},{"Add":{"text":" ~Finally() { fun(); }","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:545"}},{"Add":{"text":"};","pos":"DIEY5USNXFN6O2ARZ5IPBS2VX4OH7KE4RMX2JWAWTVMMSYA6GREAC:548"}}]}}],"owner":"grahamc","repo":"hydra","token":"TdFXyHTAkld8niDEP0hJKa16SMvS5dA2FV449P2MCtI=","login":null}

…

Dependencies

Change contents

Review