{"header":{"message":"getQueuedBuilds(): Handle dependent builds first","description":"If a build A depends on a derivation that is the top-level derivation\nof some build B, then we should process B before A (meaning we\nshouldn't make the derivation runnable before B has been\nadded). Otherwise, the derivation will be \"accounted\" to A rather than\nB (so the build step will show up in the wrong build).","timestamp":"2015-06-17T12:46:02Z","authors":[{"email":"eelco.dolstra@logicblox.com","name":"Eelco Dolstra"}]},"hash":"JAUB2FT5SNQWXR24TWTOHVT6UUJ6OQPAFPBZIQAQBHVWHWWFHLYAC","authors":[],"deps":{"hashes":[["JAUB2FT5SNQWXR24TWTOHVT6UUJ6OQPAFPBZIQAQBHVWHWWFHLYAC","getQueuedBuilds(): Handle dependent builds first"],["AA",null],["HHOMBU7GGRAEXODSDY3WUHQGOSQ35OTGRNBWKKAS2D4YEIZTTNUAC","hydra-queue-runner: Implement timeouts"],["IWB3F4Z6QZYHQFJ6FWZTGWLCPBYEUTFLUS3F7QT7JOA4DV4YLYGAC","Fail builds with previously failed steps early"],["NJJ7H64SZOX5EGACDCQAUQ7R6UEWD5IIC35A2MWFOOJV55DJYPHAC","Very basic multi-threaded queue runner"],["FQQRJUO4C7655SFAKPRPILALVEVRQSIIVBLMQUFPODUBXPIMWYSAC","Mark builds as busy"],["ENXUSMSVOU3AZFMH2ZXR4ZVPV2LRRQYQJ6IFX33YN6IH2ORSNSAAC","Make concurrency more robust"],["N5O7VEEOY2IE27VCOYRBG7YCY3K7JMQEDEMRT4OQ2MUE3NWULHHQC","Immediately abort builds that require an unsupported system type"],["GKZN4UV75GV7GEHQGBM2O6UHN6CVSHA7BMASCUQSDIDYVUKKZL7AC","Make the queue monitor more robust, and better debug output"],["24BMQDZAWDQ7VNIA7TIROXSOYLOJBNZ2E4264WHWNJAEN6ZB3UOAC","Start of single-process hydra-queue-runner"],["2IQRXLWETU2RLXPYKBSZIUULDHS346BAHE2NLOMOWXC6WDAX2BYQC","Support cancelling builds"],["22LDPAIPRVSZVZXEDCM54GUOH72VQ52EIY47PQV2VELZ3NORC5IQC","Check non-runnable steps for unsupported system type"],["UQQ4IL55WHYMXNSPOXEFBTZAPMP7LQ726THOR7INRCJDSYVOP3ZAC","Add a error type for \"unsupported system type\""],["5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","Basic remote building"],["YZAI5GQU3HNMK5MEGF2Y7WS445AN4YKD3HNJQVQP545ODN3F5DLAC","Implement a database connection pool"],["C6HOMHZWMSC7ORGFUF5YG2ACKV2SCP26HL3UH6VXH6RNDYRXH5DAC","Don't try to handle SIGINT"]],"deps":["HHOMBU7GGRAEXODSDY3WUHQGOSQ35OTGRNBWKKAS2D4YEIZTTNUAC"],"known":["24BMQDZAWDQ7VNIA7TIROXSOYLOJBNZ2E4264WHWNJAEN6ZB3UOAC","NJJ7H64SZOX5EGACDCQAUQ7R6UEWD5IIC35A2MWFOOJV55DJYPHAC","ENXUSMSVOU3AZFMH2ZXR4ZVPV2LRRQYQJ6IFX33YN6IH2ORSNSAAC","YZAI5GQU3HNMK5MEGF2Y7WS445AN4YKD3HNJQVQP545ODN3F5DLAC","5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC","FQQRJUO4C7655SFAKPRPILALVEVRQSIIVBLMQUFPODUBXPIMWYSAC","C6HOMHZWMSC7ORGFUF5YG2ACKV2SCP26HL3UH6VXH6RNDYRXH5DAC","2IQRXLWETU2RLXPYKBSZIUULDHS346BAHE2NLOMOWXC6WDAX2BYQC","N5O7VEEOY2IE27VCOYRBG7YCY3K7JMQEDEMRT4OQ2MUE3NWULHHQC","UQQ4IL55WHYMXNSPOXEFBTZAPMP7LQ726THOR7INRCJDSYVOP3ZAC","22LDPAIPRVSZVZXEDCM54GUOH72VQ52EIY47PQV2VELZ3NORC5IQC","IWB3F4Z6QZYHQFJ6FWZTGWLCPBYEUTFLUS3F7QT7JOA4DV4YLYGAC","GKZN4UV75GV7GEHQGBM2O6UHN6CVSHA7BMASCUQSDIDYVUKKZL7AC"]},"hunks":[{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":482,"ins":{"up_context":[{"change":3,"pos":1741}],"down_context":[{"change":3,"pos":1798}],"flag":"BLOCK","start":0,"end":47,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":1741},"to":{"change":3,"start":1741,"end":1798},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" std::multimap newBuilds;"}],"del_contents":[{"Del":" std::list newBuilds; // FIXME: use queue\n"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":501,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":2,"pos":715},"to":{"change":2,"start":715,"end":795},"introduced_by":2}],"inode":{"change":3,"pos":4840}}},"contents":[{"Del":" std::cerr << build->id << \" \" << build->buildTimeout << std::endl;\n"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":502,"ins":{"up_context":[{"change":3,"pos":10057}],"down_context":[{"change":3,"pos":2491}],"flag":"BLOCK","start":48,"end":118,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":10057},"to":{"change":3,"start":2451,"end":2491},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" newBuilds.emplace(std::make_pair(build->drvPath, build));"}],"del_contents":[{"Del":" newBuilds.push_back(build);\n"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":506,"ins":{"up_context":[{"change":3,"pos":10236}],"down_context":[{"change":3,"pos":10351}],"flag":"BLOCK","start":119,"end":231,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":10236},"to":{"change":3,"start":2508,"end":2856},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" std::set newRunnable;"},{"Add":" unsigned int nrAdded;"},{"Add":" std::function createBuild;"}],"del_contents":[{"Del":" /* Now instantiate build steps for each new build. The builder\n threads can start building the runnable build steps right away,\n even while we're still processing other new builds. */\n for (auto & build : newBuilds) {\n // FIXME: remove build from newBuilds to ensure quick destruction\n // FIXME: exception handling\n"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":510,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":10352}],"down_context":[{"change":3,"pos":870}],"flag":"BLOCK","start":232,"end":274,"inode":{"change":3,"pos":4840}}},"contents":[{"Add":" createBuild = [&](Build::ptr build) {"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":512,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":970}],"down_context":[{"change":3,"pos":10473}],"flag":"BLOCK","start":275,"end":294,"inode":{"change":3,"pos":4840}}},"contents":[{"Add":" nrAdded++;"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":525,"ins":{"up_context":[{"change":3,"pos":10994}],"down_context":[{"change":3,"pos":11016}],"flag":"BLOCK","start":295,"end":315,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":10994},"to":{"change":3,"start":10994,"end":11016},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" return;"}],"del_contents":[{"Del":" continue;\n"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":528,"ins":{"up_context":[{"change":3,"pos":11027}],"down_context":[{"change":3,"pos":128}],"flag":"BLOCK","start":316,"end":354,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":11027},"to":{"change":3,"start":77,"end":128},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" std::set newSteps;"}],"del_contents":[{"Del":" std::set newSteps, newRunnable;\n"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":530,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":211}],"down_context":[{"change":3,"pos":3067}],"flag":"BLOCK","start":355,"end":923,"inode":{"change":3,"pos":4840}}},"contents":[{"Add":""},{"Add":" /* Some of the new steps may be the top level of builds that"},{"Add":" we haven't processed yet. So do them now. This ensures that"},{"Add":" if build A depends on build B with top-level step X, then X"},{"Add":" will be \"accounted\" to B in doBuildStep(). */"},{"Add":" for (auto & r : newSteps) {"},{"Add":" while (true) {"},{"Add":" auto i = newBuilds.find(r->drvPath);"},{"Add":" if (i == newBuilds.end()) break;"},{"Add":" Build::ptr b = i->second;"},{"Add":" newBuilds.erase(i);"},{"Add":" createBuild(b);"},{"Add":" }"},{"Add":" }"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":558,"ins":{"up_context":[{"change":3,"pos":273}],"down_context":[{"change":3,"pos":295}],"flag":"BLOCK","start":924,"end":944,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":273},"to":{"change":3,"start":273,"end":295},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" return;"}],"del_contents":[{"Del":" continue;\n"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":606,"ins":{"up_context":[{"change":3,"pos":11403}],"down_context":[{"change":3,"pos":11435}],"flag":"BLOCK","start":945,"end":974,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":11403},"to":{"change":3,"start":1201,"end":1232},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" if (badStep) return;"}],"del_contents":[{"Del":" if (badStep) continue;\n"}]}},{"Replacement":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":619,"ins":{"up_context":[{"change":3,"pos":3558}],"down_context":[{"change":3,"pos":10110}],"flag":"BLOCK","start":975,"end":1125,"inode":{"change":3,"pos":4840}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":3558},"to":{"change":3,"start":1161,"end":1275},"introduced_by":3},{"previous":"","flag":"DELETED","from":{"change":3,"pos":1275},"to":{"change":3,"start":361,"end":442},"introduced_by":3},{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":361},"to":{"change":3,"start":361,"end":442},"introduced_by":3}],"inode":{"change":3,"pos":4840}},"ins_contents":[{"Add":" printMsg(lvlChatty, format(\"added build %1% (top-level step %2%, %3% new steps)\")"},{"Add":" % build->id % step->drvPath % newSteps.size());"}],"del_contents":[{"Del":" printMsg(lvlChatty, format(\"added build %1% (top-level step %2%, %3% new steps, %4% new runnable steps)\")\n"},{"Del":" % build->id % step->drvPath % newSteps.size() % newRunnable.size());\n"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":627,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":3806}],"down_context":[{"change":3,"pos":3806}],"flag":"BLOCK","start":1126,"end":1539,"inode":{"change":3,"pos":4840}}},"contents":[{"Add":" };"},{"Add":""},{"Add":" /* Now instantiate build steps for each new build. The builder"},{"Add":" threads can start building the runnable build steps right away,"},{"Add":" even while we're still processing other new builds. */"},{"Add":" while (!newBuilds.empty()) {"},{"Add":" auto build = newBuilds.begin()->second;"},{"Add":" newBuilds.erase(newBuilds.begin());"},{"Add":""},{"Add":" newRunnable.clear();"},{"Add":" nrAdded = 0;"},{"Add":" createBuild(build);"},{"Add":""}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":642,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":3915}],"down_context":[{"change":3,"pos":3915}],"flag":"BLOCK","start":1540,"end":1658,"inode":{"change":3,"pos":4840}}},"contents":[{"Add":" printMsg(lvlChatty, format(\"got %1% new runnable steps from %2% new builds\") % newRunnable.size() % nrAdded);"}]}},{"Edit":{"inode":{"change":3,"pos":4840},"path":"src/hydra-queue-runner/hydra-queue-runner.cc","line":1264,"atom":{"EdgeMap":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":14778},"to":{"change":2,"start":1601,"end":1616},"introduced_by":2}],"inode":{"change":3,"pos":4840}}},"contents":[{"Del":" sleep(5);\n\n"}]}}],"owner":"grahamc","repo":"hydra2","token":"Y+upM61KYxUQ+Zh7y0NXe4ZxwpvYt79EvLQxYAReJpY=","login":null}

…

Dependencies

Change contents