grahamc/hydra - Change VQISTKOPNAEUS2K2F73CMNNLGZATWUYIURD5CSVNBNF7Q5ZF4PXQC

hydra-queue-runner: Use substitutes

This allows Hydra to use binaries from available binary caches. It makes the queue monitor thread quite a bit slower, so if you don't want to use binary caches, it's better to add "–option build-use-substitutes false" to the hydra-queue-runner invocation.

Fixed #243.

Created by Eelco Dolstra on October 5, 2015

VQISTKOPNAEUS2K2F73CMNNLGZATWUYIURD5CSVNBNF7Q5ZF4PXQC

Dependencies

In channels

main

upstream

Change contents

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 171 [4.4840]

B:BD[4.8075] → [4.8075:8172]

B:BD[4.8172] → [5.263:374]

int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,
    const std::string & machine, BuildStepStatus status, const std::string & errorMsg, BuildID propagatedFrom)

[4.8075]

[4.8254]

int State::allocBuildStep(pqxx::work & txn, Build::ptr build)

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 178 [4.4840]

B:BD[4.8363] → [4.8363:8431]

    int stepNr = res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1;

[4.8363]

[4.8431]

    return res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1;
}
int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step,
    const std::string & machine, BuildStepStatus status, const std::string & errorMsg, BuildID propagatedFrom)
{
    int stepNr = allocBuildStep(txn, build);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 189 [4.4840]

B:BD[5.578] → [6.0:73]

        (build->id)(stepNr)(0)(step->drvPath)(status == bssBusy ? 1 : 0)

[5.578]

[6.73]

        (build->id)
        (stepNr)
        (0) // == build
        (step->drvPath)
        (status == bssBusy ? 1 : 0)

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 226 [4.4840]

[4.13425]

[7.2306]

int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
    Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath)
{
    int stepNr = allocBuildStep(txn, build);
    txn.parameterized
        ("insert into BuildSteps (build, stepnr, type, drvPath, busy, status, startTime, stopTime) values ($1, $2, $3, $4, $5, $6, $7, $8)")
        (build->id)
        (stepNr)
        (1) // == substitution
        (drvPath)
        (0)
        (0)
        (startTime)
        (stopTime).exec();
    txn.parameterized
        ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
        (build->id)(stepNr)(outputName)(storePath).exec();
    return stepNr;
}

Deletion in src/hydra-queue-runner/hydra-queue-runner.cc at line 724 [4.4840]
B:BD[4.19221] → [4.19221:19262]
```
        settings.useSubstitutes = false;
```
Insertion in src/hydra-queue-runner/queue-monitor.cc at line 3 [3.20067]
[3.20115]
[3.20115]
```
#include "globals.hh"
```
Insertion in src/hydra-queue-runner/queue-monitor.cc at line 5 [3.20067]
[3.20116]
[3.20116]

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 136 [3.20067]

B:BD[3.24028] → [3.24028:24135]

        Step::ptr step = createStep(store, build->drvPath, build, 0, finishedDrvs, newSteps, newRunnable);

[3.24028]

[3.24135]

        Step::ptr step = createStep(store, conn, build, build->drvPath, build, 0, finishedDrvs, newSteps, newRunnable);

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 303 [3.20067]

B:BD[3.29402] → [3.29402:29485]

Step::ptr State::createStep(std::shared_ptr<StoreAPI> store, const Path & drvPath,

[3.29402]

[3.29485]

Step::ptr State::createStep(std::shared_ptr<StoreAPI> store,
    Connection & conn, Build::ptr build, const Path & drvPath,

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 382 [3.20067]

B:BD[3.31904] → [3.31904:31945]

    for (auto & i : step->drv.outputs) {

[3.31904]

[3.31945]

    PathSet outputs = outputPaths(step->drv);
    DerivationOutputs missing;
    PathSet missingPaths;
    for (auto & i : step->drv.outputs)

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 388 [3.20067]

B:BD[3.32022] → [3.32022:32041]

            break;

[3.32022]

[3.32041]

            missing[i.first] = i.second;
            missingPaths.insert(i.second.path);
        }
    /* Try to substitute the missing paths. Note: can't use the more
       efficient querySubstitutablePaths() here because upstream Hydra
       servers don't allow it (they have "WantMassQuery: 0"). */
    assert(missing.size() == missingPaths.size());
    if (!missing.empty() && settings.useSubstitutes) {
        SubstitutablePathInfos infos;
        store->querySubstitutablePathInfos(missingPaths, infos);
        if (infos.size() == missingPaths.size()) {
            valid = true;
            for (auto & i : missing) {
                try {
                    printMsg(lvlInfo, format("substituting output ‘%1%’ of ‘%2%’") % i.second.path % drvPath);
                    time_t startTime = time(0);
                    store->ensurePath(i.second.path);
                    time_t stopTime = time(0);
                    {
                        pqxx::work txn(conn);
                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path);
                        txn.commit();
                    }
                } catch (Error & e) {
                    valid = false;
                    break;
                }
            }

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 435 [3.20067]

B:BD[3.32426] → [3.32426:32519]

        auto dep = createStep(store, i.first, 0, step, finishedDrvs, newSteps, newRunnable);

[3.32426]

[3.32519]

        auto dep = createStep(store, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);

Insertion in src/hydra-queue-runner/state.hh at line 359 [8.1128]
[8.7026]
[8.7026]
```
    int allocBuildStep(pqxx::work & txn, Build::ptr build);
```

Insertion in src/hydra-queue-runner/state.hh at line 369 [8.1128]

[8.7490]


    int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
        Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath);

Replacement in src/hydra-queue-runner/state.hh at line 385 [8.1128]

B:BD[8.7785] → [2.876:966]

    Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store, const nix::Path & drvPath,

[8.7785]

[2.966]

    Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store,
        Connection & conn, Build::ptr build, const nix::Path & drvPath,