{"header":{"message":"buildRemote(): Copy paths to the destination store in O(1) memory","description":null,"timestamp":"2020-07-27T16:11:04Z","authors":[{"email":"edolstra@gmail.com","name":"Eelco Dolstra"}]},"hash":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC","authors":[],"deps":{"hashes":[["5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC","buildRemote(): Copy paths to the destination store in O(1) memory"],["AA",null],["H46LUUZY3N7DDOIZFDZ7SJP76BICH4G6Z4SC7GBPMIKHW2JPZRTAC","hydra-queue-runner: don't try to distribute builds on localhost"],["BTSZJ7W2DGNGOOBOMTTZTUOAQ3IYMCYZLT2MJ7C3UZX5KSOZ4ZVAC","Quick hack to fix compilation"],["BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC","Temporarily disable machines on any exception, not just connection failures"],["6EO3HVNAAVJLBYTBSHXQADDTYF7JJU3ANHS5SKHSSROOW4AXIZ4AC","Merge remote-tracking branch 'origin/master' into binary-cache"],["BUEWVH2MOUKUGR4F2BOHIUP63A7UYPLPYBRLJQY6HW4T2HF3LHYQC","Remove signing parameter (nix#f435f82)"],["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","Basic remote building"],["SL3WSRACCX2IMJHHLTRAUQT7QDLCOKYLVO2FEHWIHXM5GPKSRJTQC","hydra-queue-runner: Limit memory usage"],["FITVNQ2SVM6KSOF5P3HHWJYQ3WMQYDJGAONCBIZ7OF7CPXGMA36QC","Keep track of the time we spend copying to/from build machines"],["WV4SSAIYM4SVBQ2VISDTAXQJCPKRGTLSVFH44CQFEMC4COWG5OKQC","Build against nix-master"],["EBJP3MNAC4CRYGMJJHMKUYWTXP3N4WMMORRLDSDHQVF5ZLOTIENQC","Build against nix-master"],["UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC","Prevent download of NARs we just uploaded"],["PW4TLMWSMKGTN4UKRNQD3H2PWLTXOPSXDZXUVVFQSQS6AFBQLDDAC","Sync with latest Nix"],["EFWDY2V7ILJBFQ6NADBESYZTPHQBXKNO5N4DPNABL7X4XMNNTNYAC","Support testing build determinism"],["LE4VZIY5VZ52FOP5QQRIJINWIMWTAPRTZTGO77JXUEPGRPRSQYMAC","More stats"],["5CNQUQ77EWOYWMW6VXQCSDITIO55UEKBI6CBV72NR7DPUBFLMWDAC","Bump Nix"],["73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","hydra-queue-runner: Write directly to a binary cache"]],"deps":["H46LUUZY3N7DDOIZFDZ7SJP76BICH4G6Z4SC7GBPMIKHW2JPZRTAC","BTSZJ7W2DGNGOOBOMTTZTUOAQ3IYMCYZLT2MJ7C3UZX5KSOZ4ZVAC"],"known":["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","LE4VZIY5VZ52FOP5QQRIJINWIMWTAPRTZTGO77JXUEPGRPRSQYMAC","FITVNQ2SVM6KSOF5P3HHWJYQ3WMQYDJGAONCBIZ7OF7CPXGMA36QC","6EO3HVNAAVJLBYTBSHXQADDTYF7JJU3ANHS5SKHSSROOW4AXIZ4AC","UVNTWTWGQOFKDAJ2ROJYT4U2N4EUXKNWZWPHOM42WPLUL4ALXRJQC","SL3WSRACCX2IMJHHLTRAUQT7QDLCOKYLVO2FEHWIHXM5GPKSRJTQC","BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC","BUEWVH2MOUKUGR4F2BOHIUP63A7UYPLPYBRLJQY6HW4T2HF3LHYQC","5CNQUQ77EWOYWMW6VXQCSDITIO55UEKBI6CBV72NR7DPUBFLMWDAC","EFWDY2V7ILJBFQ6NADBESYZTPHQBXKNO5N4DPNABL7X4XMNNTNYAC","PW4TLMWSMKGTN4UKRNQD3H2PWLTXOPSXDZXUVVFQSQS6AFBQLDDAC","WV4SSAIYM4SVBQ2VISDTAXQJCPKRGTLSVFH44CQFEMC4COWG5OKQC","EBJP3MNAC4CRYGMJJHMKUYWTXP3N4WMMORRLDSDHQVF5ZLOTIENQC"]},"hunks":[{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":127,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":2443}],"down_context":[{"change":18,"pos":544}],"flag":"BLOCK","start":0,"end":864,"inode":{"change":4,"pos":85}}},"contents":[{"Add":{"text":"// FIXME: use Store::topoSortPaths().","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:37"}},{"Add":{"text":"StorePaths topoSortPaths(const std::map & paths)","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:112"}},{"Add":{"text":"{","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:114"}},{"Add":{"text":" StorePaths sorted;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:137"}},{"Add":{"text":" StorePathSet visited;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:163"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:164"}},{"Add":{"text":" std::function dfsVisit;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:222"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:223"}},{"Add":{"text":" dfsVisit = [&](const StorePath & path) {","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:268"}},{"Add":{"text":" if (!visited.insert(path).second) return;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:318"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:319"}},{"Add":{"text":" auto info = paths.find(path);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:357"}},{"Add":{"text":" auto references = info == paths.end() ? StorePathSet() : info->second.references;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:447"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:448"}},{"Add":{"text":" for (auto & i : references)","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:484"}},{"Add":{"text":" /* Don't traverse into paths that don't exist. That can","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:553"}},{"Add":{"text":" happen due to substitutes for non-existent paths. */","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:621"}},{"Add":{"text":" if (i != path && paths.count(i))","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:666"}},{"Add":{"text":" dfsVisit(i);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:695"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:696"}},{"Add":{"text":" sorted.push_back(path);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:728"}},{"Add":{"text":" };","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:735"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:736"}},{"Add":{"text":" for (auto & i : paths)","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:763"}},{"Add":{"text":" dfsVisit(i.first);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:790"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:791"}},{"Add":{"text":" std::reverse(sorted.begin(), sorted.end());","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:839"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:840"}},{"Add":{"text":" return sorted;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:859"}},{"Add":{"text":"}","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:861"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:862"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:863"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":183,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":3316}],"down_context":[{"change":4,"pos":11}],"flag":"BLOCK","start":865,"end":905,"inode":{"change":4,"pos":85}}},"contents":[{"Add":{"text":" // FIXME: rewrite to use Store.","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:904"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":218,"ins":{"up_context":[{"change":4,"pos":415}],"down_context":[{"change":4,"pos":457}],"flag":"BLOCK","start":906,"end":948,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":415},"to":{"change":4,"start":0,"end":42},"introduced_by":4}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" to << SERVE_MAGIC_1 << 0x204;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:947"}}],"del_contents":[{"Del":{"text":" to << SERVE_MAGIC_1 << 0x203;","pos":"EFWDY2V7ILJBFQ6NADBESYZTPHQBXKNO5N4DPNABL7X4XMNNTNYAC:41"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":441,"ins":{"up_context":[{"change":4,"pos":47}],"down_context":[{"change":4,"pos":7356}],"flag":"BLOCK","start":949,"end":1054,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":47},"to":{"change":4,"start":7302,"end":7356},"introduced_by":4}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" /* Get info about each output path. */","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:999"}},{"Add":{"text":" std::map infos;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1053"}}],"del_contents":[{"Del":{"text":" /* Query the size of the output paths. */","pos":"BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC:7355"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":448,"ins":{"up_context":[{"change":4,"pos":7492}],"down_context":[{"change":4,"pos":7543}],"flag":"BLOCK","start":1055,"end":1278,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":7492},"to":{"change":4,"start":7492,"end":7543},"introduced_by":4}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" auto storePathS = readString(from);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1106"}},{"Add":{"text":" if (storePathS == \"\") break;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1151"}},{"Add":{"text":" ValidPathInfo info(localStore->parseStorePath(storePathS));","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1227"}},{"Add":{"text":" assert(outputs.count(info.path));","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1277"}}],"del_contents":[{"Del":{"text":" if (readString(from) == \"\") break;","pos":"BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC:7542"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":453,"ins":{"up_context":[{"change":4,"pos":7588}],"down_context":[{"change":4,"pos":7646}],"flag":"BLOCK","start":1279,"end":1362,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":7588},"to":{"change":4,"start":7588,"end":7646},"introduced_by":4}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" info.references = readStorePaths(*localStore, from);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1361"}}],"del_contents":[{"Del":{"text":" readStrings(from); // references","pos":"BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC:7645"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":455,"ins":{"up_context":[{"change":4,"pos":7699}],"down_context":[{"change":4,"pos":7751}],"flag":"BLOCK","start":1363,"end":1704,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":7699},"to":{"change":4,"start":7699,"end":7751},"introduced_by":4}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" info.narSize = readLongLong(from);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1413"}},{"Add":{"text":" totalNarSize += info.narSize;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1459"}},{"Add":{"text":" info.narHash = Hash(readString(from), htSHA256);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1524"}},{"Add":{"text":" info.ca = parseContentAddressOpt(readString(from));","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1592"}},{"Add":{"text":" readStrings(from); // sigs","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1646"}},{"Add":{"text":" infos.insert_or_assign(info.path, info);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1703"}}],"del_contents":[{"Del":{"text":" totalNarSize += readLongLong(from);","pos":"BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC:7750"}}]}},{"Edit":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":468,"atom":{"NewVertex":{"up_context":[{"change":4,"pos":104}],"down_context":[{"change":4,"pos":2796}],"flag":"BLOCK","start":1705,"end":1739,"inode":{"change":4,"pos":85}}},"contents":[{"Add":{"text":" /* Copy each path. */","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1738"}}]}},{"Replacement":{"inode":{"change":4,"pos":85},"path":"src/hydra-queue-runner/build-remote.cc","line":472,"ins":{"up_context":[{"change":4,"pos":218}],"down_context":[{"change":4,"pos":231}],"flag":"BLOCK","start":1740,"end":2065,"inode":{"change":4,"pos":85}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":218},"to":{"change":4,"start":2648,"end":2742},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":2742},"to":{"change":2,"start":92,"end":92},"introduced_by":2},{"previous":"","flag":"DELETED","from":{"change":2,"pos":92},"to":{"change":4,"start":8846,"end":8870},"introduced_by":2},{"previous":"","flag":"DELETED","from":{"change":4,"pos":2742},"to":{"change":4,"start":8846,"end":8870},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":3249},"to":{"change":4,"start":8846,"end":8870},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":8846},"to":{"change":4,"start":8846,"end":8870},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":8870},"to":{"change":3,"start":0,"end":78},"introduced_by":3}],"inode":{"change":4,"pos":85}},"ins_contents":[{"Add":{"text":" auto pathsSorted = topoSortPaths(infos);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1792"}},{"Add":{"text":"","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1793"}},{"Add":{"text":" for (auto & path : pathsSorted) {","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1839"}},{"Add":{"text":" auto & info = infos.find(path)->second;","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1895"}},{"Add":{"text":" to << cmdDumpStorePath << localStore->printStorePath(path);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1971"}},{"Add":{"text":" to.flush();","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:1999"}},{"Add":{"text":" destStore->addToStore(info, from);","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:2050"}},{"Add":{"text":" }","pos":"5JMO43B45WLRDAZJ4QIDFHSI7BVESV7CJ4BYXPJ5Q3MILZ23SECAC:2064"}}],"del_contents":[{"Del":{"text":" to << cmdExportPaths << 0;","pos":"WV4SSAIYM4SVBQ2VISDTAXQJCPKRGTLSVFH44CQFEMC4COWG5OKQC:2686"}},{"Del":{"text":" writeStorePaths(*localStore, to, outputs);","pos":"WV4SSAIYM4SVBQ2VISDTAXQJCPKRGTLSVFH44CQFEMC4COWG5OKQC:2741"}},{"Del":{"text":"","pos":"H46LUUZY3N7DDOIZFDZ7SJP76BICH4G6Z4SC7GBPMIKHW2JPZRTAC:92"}},{"Del":{"text":" to.flush();","pos":"BYVRA54QBKHLFOPIRBJKZZI7JYBYHSOK7MIA3TUZTALZQJGG3G7QC:8869"}},{"Del":{"text":" destStore->importPaths(from, /* result.accessor, */ NoCheckSigs);","pos":"BTSZJ7W2DGNGOOBOMTTZTUOAQ3IYMCYZLT2MJ7C3UZX5KSOZ4ZVAC:77"}}]}}],"owner":"grahamc","repo":"hydra","token":"SMxTXnTZg35J0V3zMZyEHQfEA5q8LWghNCypZ5wa7yM=","login":null}

…

Dependencies

Change contents

Review