hydra-queue-runner: Use substitutes

[?]

Oct 5, 2015, 12:57 PM

VQISTKOPNAEUS2K2F73CMNNLGZATWUYIURD5CSVNBNF7Q5ZF4PXQC

Dependencies

[2] OG3Z3QGC Namespace cleanup
[3] MHVIT4JY Split hydra-queue-runner.cc more
[4] KBZHIGLG Record the machine used for a build step
[5] 5N7LVAJN Keep track of requiredSystemFeatures in the machine stats
[6] VRDAPUA4 Don't lock the CPU
[7] JFD25IUU hydra-queue-runner: Implement --unlock
[8] 24BMQDZA Start of single-process hydra-queue-runner
[9] ZK76B5ZZ Load the queue in order of global priority
[10] IK2UBDAU Revive jobset scheduling
[11] HJOEIMLR Refactor
[12] FQQRJUO4 Mark builds as busy
[13] WE5Q2NVI Allow build to be bumped to the front of the queue via the web interface
[14] EJJCY263 Account steps with preferLocalBuild as a separate system type
[15] 62MQPRXC Pass null values to libpqxx properly
[16] NJJ7H64S Very basic multi-threaded queue runner
[17] LJILHOJ7 Create BuildSteps race-free
[18] MSIHMO45 Tweak build steps
[*] HUUZFPPK Fix race between the queue monitor and the builder threads

Change contents

replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 171

[4.8075]→[4.8075:8172](∅→∅),[4.8172]→[4.263:374](∅→∅)

int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,
const std::string & machine, BuildStepStatus status, const std::string & errorMsg, BuildID propagatedFrom)

[4.8075]

[4.8254]

int State::allocBuildStep(pqxx::work & txn, Build::ptr build)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 178

[4.8363]→[4.8363:8431](∅→∅)

int stepNr = res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1;

[4.8363]

[4.8431]

return res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1;
}

int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,
const std::string & machine, BuildStepStatus status, const std::string & errorMsg, BuildID propagatedFrom)
{
int stepNr = allocBuildStep(txn, build);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 189

[4.578]→[4.0:73](∅→∅)

(build->id)(stepNr)(0)(step->drvPath)(status == bssBusy ? 1 : 0)

[4.578]

[4.73]

(build->id)
(stepNr)
(0) // == build
(step->drvPath)
(status == bssBusy ? 1 : 0)
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 226

[4.13425]

[20.2306]

int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath)
{
int stepNr = allocBuildStep(txn, build);

txn.parameterized
("insert into BuildSteps (build, stepnr, type, drvPath, busy, status, startTime, stopTime) values ($1, $2, $3, $4, $5, $6, $7, $8)")
(build->id)
(stepNr)
(1) // == substitution
(drvPath)
(0)
(0)
(startTime)
(stopTime).exec();

txn.parameterized
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
(build->id)(stepNr)(outputName)(storePath).exec();

return stepNr;
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 724

[4.19221]→[4.19221:19262](∅→∅)

settings.useSubstitutes = false;
edit in src/hydra-queue-runner/queue-monitor.cc at line 3

[3.20115]

[3.20115]

#include "globals.hh"
edit in src/hydra-queue-runner/queue-monitor.cc at line 5

[3.20116]

[3.20116]
replacement in src/hydra-queue-runner/queue-monitor.cc at line 136

[3.24028]→[3.24028:24135](∅→∅)

Step::ptr step = createStep(store, build->drvPath, build, 0, finishedDrvs, newSteps, newRunnable);

[3.24028]

[3.24135]

Step::ptr step = createStep(store, conn, build, build->drvPath, build, 0, finishedDrvs, newSteps, newRunnable);
replacement in src/hydra-queue-runner/queue-monitor.cc at line 303

[3.29402]→[3.29402:29485](∅→∅)

Step::ptr State::createStep(std::shared_ptr<StoreAPI> store, const Path & drvPath,

[3.29402]

[3.29485]

Step::ptr State::createStep(std::shared_ptr<StoreAPI> store,
Connection & conn, Build::ptr build, const Path & drvPath,
replacement in src/hydra-queue-runner/queue-monitor.cc at line 382

[3.31904]→[3.31904:31945](∅→∅)

for (auto & i : step->drv.outputs) {

[3.31904]

[3.31945]

PathSet outputs = outputPaths(step->drv);
DerivationOutputs missing;
PathSet missingPaths;
for (auto & i : step->drv.outputs)
replacement in src/hydra-queue-runner/queue-monitor.cc at line 388

[3.32022]→[3.32022:32041](∅→∅)

break;

[3.32022]

[3.32041]

missing[i.first] = i.second;
missingPaths.insert(i.second.path);
}

/* Try to substitute the missing paths. Note: can't use the more
efficient querySubstitutablePaths() here because upstream Hydra
servers don't allow it (they have "WantMassQuery: 0"). */
assert(missing.size() == missingPaths.size());
if (!missing.empty() && settings.useSubstitutes) {
SubstitutablePathInfos infos;
store->querySubstitutablePathInfos(missingPaths, infos);
if (infos.size() == missingPaths.size()) {
valid = true;
for (auto & i : missing) {
try {
printMsg(lvlInfo, format("substituting output ‘%1%’ of ‘%2%’") % i.second.path % drvPath);

time_t startTime = time(0);
store->ensurePath(i.second.path);
time_t stopTime = time(0);

{
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path);
txn.commit();
}

} catch (Error & e) {
valid = false;
break;
}
}
replacement in src/hydra-queue-runner/queue-monitor.cc at line 435

[3.32426]→[3.32426:32519](∅→∅)

auto dep = createStep(store, i.first, 0, step, finishedDrvs, newSteps, newRunnable);

[3.32426]

[3.32519]

auto dep = createStep(store, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
edit in src/hydra-queue-runner/state.hh at line 359

[4.7026]

[4.7026]

int allocBuildStep(pqxx::work & txn, Build::ptr build);
edit in src/hydra-queue-runner/state.hh at line 369

[4.7490]

[4.7490]

int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath);
replacement in src/hydra-queue-runner/state.hh at line 385

[4.7785]→[2.876:966](∅→∅)

Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store, const nix::Path & drvPath,

[4.7785]

[2.966]

Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store,
Connection & conn, Build::ptr build, const nix::Path & drvPath,