{"header":{"message":"Reuse build products / metrics stored in the database","description":"Previously, if the queue monitor thread encounters a build that Hydra\nhas previously built, it downloaded the output paths from the binary\ncache, just to determine the build products and metrics. This is very\ninefficient. In particular, when doing something like merging\nnixpkgs:staging into nixpkgs:master, the queue monitor thread will be\nlocked up for a long time fetching files from S3, causing the build\nfarm to be mostly idle.\n\nOf course this is entirely unnecessary, since the build\nproducts/metrics are already in the Hydra database. So now we just\nlook up a previous build with the same output path, and copy the\nproducts/metrics.","timestamp":"2016-04-13T13:38:06Z","authors":[{"email":"eelco.dolstra@logicblox.com","name":"Eelco Dolstra"}]},"hash":"WDGARQ76X6RLSFPJTW52BZYFKBC7DPSTABI7HHIKNYHBYXSUNTHQC","authors":[],"deps":{"hashes":[["WDGARQ76X6RLSFPJTW52BZYFKBC7DPSTABI7HHIKNYHBYXSUNTHQC","Reuse build products / metrics stored in the database"],["AA",null],["ACBS7C6QGXAMPLQ336WAQMMGLZXBDNNYPE26ZBPUQ24GL4BF643AC","hydra-queue-runner: Detect changes to the scheduling shares"],["BAFICF73XUYNAYT6T6X7GQIFHG6CLGPYYNFBMC6ON6SVPNOXO6RAC","Support hydra-build-products on binary cache stores"],["DKJFD6JNNK5LJMRGQABMJZKMFZLGY3ADKJWF6J4BBHEUPY3NB67QC","Process Nix API changes"],["H7SZRHUBSEDJ3PQ3VOV4ZEEFUWTBPGZN53G6W26HHXPJDO7MSHJAC","Use nix::willBuildLocally()"],["VQISTKOPNAEUS2K2F73CMNNLGZATWUYIURD5CSVNBNF7Q5ZF4PXQC","hydra-queue-runner: Use substitutes"],["MHVIT4JYWUYD4UCGB2AHLXWLX6B5SYE22BREERNGANT7RGGDUFOAC","Split hydra-queue-runner.cc more"],["ZK76B5ZZCEOJQUDOVKDZPE6IN2YIXQONFUIJ633OAXNC73OG3YSAC","Load the queue in order of global priority"],["73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC","hydra-queue-runner: Write directly to a binary cache"],["HJOEIMLRDVQ2KZI5HGL2HKGBM3AHP7YIKGKDAGFUNKRUXVRB24NAC","Refactor"]],"deps":["ACBS7C6QGXAMPLQ336WAQMMGLZXBDNNYPE26ZBPUQ24GL4BF643AC","BAFICF73XUYNAYT6T6X7GQIFHG6CLGPYYNFBMC6ON6SVPNOXO6RAC"],"known":["MHVIT4JYWUYD4UCGB2AHLXWLX6B5SYE22BREERNGANT7RGGDUFOAC","ZK76B5ZZCEOJQUDOVKDZPE6IN2YIXQONFUIJ633OAXNC73OG3YSAC","VQISTKOPNAEUS2K2F73CMNNLGZATWUYIURD5CSVNBNF7Q5ZF4PXQC","H7SZRHUBSEDJ3PQ3VOV4ZEEFUWTBPGZN53G6W26HHXPJDO7MSHJAC","DKJFD6JNNK5LJMRGQABMJZKMFZLGY3ADKJWF6J4BBHEUPY3NB67QC","73YR46NJNYZQKHA3QDJCAZYAKC2CGEF5LIS44NOIPDZU6FX6BDPQC"]},"hunks":[{"Edit":{"inode":{"change":4,"pos":20067},"path":"src/hydra-queue-runner/queue-monitor.cc","line":4,"atom":{"EdgeMap":{"edges":[{"previous":"","flag":"DELETED","from":{"change":4,"pos":77},"to":{"change":4,"start":20115,"end":20116},"introduced_by":4},{"previous":"","flag":"DELETED","from":{"change":4,"pos":1279},"to":{"change":4,"start":20115,"end":20116},"introduced_by":4},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":20115},"to":{"change":4,"start":20115,"end":20116},"introduced_by":4}],"inode":{"change":4,"pos":20067}}},"contents":[{"Del":"\n"}]}},{"Replacement":{"inode":{"change":4,"pos":20067},"path":"src/hydra-queue-runner/queue-monitor.cc","line":161,"ins":{"up_context":[{"change":4,"pos":24924}],"down_context":[{"change":4,"pos":24982}],"flag":"BLOCK","start":0,"end":74,"inode":{"change":4,"pos":20067}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":4,"pos":24924},"to":{"change":3,"start":2070,"end":2160},"introduced_by":3}],"inode":{"change":4,"pos":20067}},"ins_contents":[{"Add":" BuildOutput res = getBuildOutputCached(conn, destStore, drv);"}],"del_contents":[{"Del":" BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv);\n"}]}},{"Edit":{"inode":{"change":4,"pos":20067},"path":"src/hydra-queue-runner/queue-monitor.cc","line":531,"atom":{"NewVertex":{"up_context":[{"change":2,"pos":1117}],"down_context":[{"change":2,"pos":1117}],"flag":"BLOCK","start":75,"end":2675,"inode":{"change":4,"pos":20067}}},"contents":[{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":""},{"Add":"BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::Derivation & drv)"},{"Add":"{"},{"Add":" {"},{"Add":" pqxx::work txn(conn);"},{"Add":""},{"Add":" for (auto & output : drv.outputs) {"},{"Add":" auto r = txn.parameterized"},{"Add":" (\"select id, buildStatus, releaseName, closureSize, size from Builds b \""},{"Add":" \"join BuildOutputs o on b.id = o.build \""},{"Add":" \"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1\")"},{"Add":" (output.second.path).exec();"},{"Add":" if (r.empty()) continue;"},{"Add":" BuildID id = r[0][0].as();"},{"Add":""},{"Add":" printMsg(lvlInfo, format(\"re-using products of build %d\") % id);"},{"Add":""},{"Add":" BuildOutput res;"},{"Add":" res.failed = r[0][1].as() == bsFailedWithOutput;"},{"Add":" res.releaseName = r[0][2].is_null() ? \"\" : r[0][2].as();"},{"Add":" res.closureSize = r[0][3].is_null() ? 0 : r[0][3].as();"},{"Add":" res.size = r[0][4].is_null() ? 0 : r[0][4].as();"},{"Add":""},{"Add":" auto products = txn.parameterized"},{"Add":" (\"select type, subtype, fileSize, sha1hash, sha256hash, path, name, defaultPath from BuildProducts where build = $1 order by productnr\")"},{"Add":" (id).exec();"},{"Add":""},{"Add":" for (auto row : products) {"},{"Add":" BuildProduct product;"},{"Add":" product.type = row[0].as();"},{"Add":" product.subtype = row[1].as();"},{"Add":" if (row[2].is_null())"},{"Add":" product.isRegular = false;"},{"Add":" else {"},{"Add":" product.isRegular = true;"},{"Add":" product.fileSize = row[2].as();"},{"Add":" }"},{"Add":" if (!row[3].is_null())"},{"Add":" product.sha1hash = parseHash(htSHA1, row[3].as());"},{"Add":" if (!row[4].is_null())"},{"Add":" product.sha256hash = parseHash(htSHA256, row[4].as());"},{"Add":" if (!row[5].is_null())"},{"Add":" product.path = row[5].as();"},{"Add":" product.name = row[6].as();"},{"Add":" if (!row[7].is_null())"},{"Add":" product.defaultPath = row[7].as();"},{"Add":" res.products.emplace_back(product);"},{"Add":" }"},{"Add":""},{"Add":" auto metrics = txn.parameterized"},{"Add":" (\"select name, unit, value from BuildMetrics where build = $1\")"},{"Add":" (id).exec();"},{"Add":""},{"Add":" for (auto row : metrics) {"},{"Add":" BuildMetric metric;"},{"Add":" metric.name = row[0].as();"},{"Add":" metric.unit = row[1].is_null() ? \"\" : row[1].as();"},{"Add":" metric.value = row[2].as();"},{"Add":" res.metrics.emplace(metric.name, metric);"},{"Add":" }"},{"Add":""},{"Add":" return res;"}]}},{"Edit":{"inode":{"change":4,"pos":20067},"path":"src/hydra-queue-runner/queue-monitor.cc","line":597,"atom":{"NewVertex":{"up_context":[{"change":2,"pos":1123}],"down_context":[{"change":2,"pos":1123}],"flag":"BLOCK","start":2676,"end":2755,"inode":{"change":4,"pos":20067}}},"contents":[{"Add":""},{"Add":" }"},{"Add":""},{"Add":" return getBuildOutput(destStore, destStore->getFSAccessor(), drv);"}]}},{"Edit":{"inode":{"change":11,"pos":1128},"path":"src/hydra-queue-runner/state.hh","line":413,"atom":{"NewVertex":{"up_context":[{"change":11,"pos":7785}],"down_context":[{"change":4,"pos":1160}],"flag":"BLOCK","start":2756,"end":2883,"inode":{"change":11,"pos":1128}}},"contents":[{"Add":" BuildOutput getBuildOutputCached(Connection & conn, nix::ref destStore,"},{"Add":" const nix::Derivation & drv);"},{"Add":""}]}}],"owner":"grahamc","repo":"hydra2","token":"dNN31C1CjBBdkfe4hI0/m3q3xabdDuuGK5nPYqrE2PQ=","login":null}

…

Dependencies

Change contents