grahamc/hydra2 - Change GKZN4UV75GV7GEHQGBM2O6UHN6CVSHA7BMASCUQSDIDYVUKKZL7AC

Make the queue monitor more robust, and better debug output

Created by Eelco Dolstra on June 15, 2015

GKZN4UV75GV7GEHQGBM2O6UHN6CVSHA7BMASCUQSDIDYVUKKZL7AC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 79 [4.4840]

B:BD[3.229] → [3.229:294]

        printMsg(lvlError, format("destroying build %1%") % id);

[3.229]

[3.294]

        printMsg(lvlDebug, format("destroying build %1%") % id);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 113 [4.4840]

B:BD[3.676] → [3.676:745]

        printMsg(lvlError, format("destroying step %1%") % drvPath);

[3.676]

[5.5921]

        printMsg(lvlDebug, format("destroying step %1%") % drvPath);

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 226 [4.4840]
[6.1]
[6.1]
```
    void queueMonitorLoop();
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 275 [4.4840]

B:BD[4.7535] → [7.38:109]

        printMsg(lvlError, "clearing active builds / build steps...");

[4.7535]

[7.109]

        printMsg(lvlInfo, "clearing active builds / build steps...");

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 387 [4.4840]

[8.226]

[9.1433]

{
    while (true) {
        try {
            queueMonitorLoop();
        } catch (std::exception & e) {
            printMsg(lvlError, format("queue monitor: %1%") % e.what());
            sleep(10); // probably a DB problem, so don't retry right away
        }
    }
}
void State::queueMonitorLoop()

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 435 [4.4840]

B:BD[10.980] → [10.980:1063]

            printMsg(lvlError, "got notification: new builds added to the queue");

[10.980]

[10.1063]

            printMsg(lvlTalkative, "got notification: new builds added to the queue");

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 437 [4.4840]

B:BD[10.1100] → [10.1100:1170]

            printMsg(lvlError, "got notification: builds restarted");

[10.1100]

[10.1170]

            printMsg(lvlTalkative, "got notification: builds restarted");

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 441 [4.4840]

B:BD[10.1266] → [10.1266:1336]

            printMsg(lvlError, "got notification: builds cancelled");

[10.1266]

[6.213]

            printMsg(lvlTalkative, "got notification: builds cancelled");

Deletion in src/hydra-queue-runner/hydra-queue-runner.cc at line 445 [4.4840]
B:BD[9.1755] → [9.1755:1803]
```
    printMsg(lvlError, "queue monitor exits");
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 450 [4.4840]

B:BD[4.9880] → [10.1427:1515]

    printMsg(lvlError, format("checking the queue for builds > %1%...") % lastBuildId);

[4.9880]

[9.1943]

    printMsg(lvlInfo, format("checking the queue for builds > %1%...") % lastBuildId);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 483 [4.4840]

B:BD[4.10352] → [3.2857:2952]

        printMsg(lvlInfo, format("loading build %1% (%2%)") % build->id % build->fullJobName);

[4.10352]

[4.10473]

        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 508 [4.4840]

B:BD[5.9878] → [5.9878:9949]

            printMsg(lvlInfo, format("cached build %1%") % build->id);

[5.9878]

[4.11213]

            printMsg(lvlInfo, format("marking build %1% as cached successful") % build->id);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 540 [4.4840]

B:BD[2.503] → [2.503:604]

                printMsg(lvlError, format("failing build %1% due to previous failure") % build->id);

[2.503]

[2.604]

                printMsg(lvlError, format("marking build %1% as cached failure") % build->id);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 574 [4.4840]

B:BD[3.3558] → [11.249:361]

        printMsg(lvlInfo, format("added build %1% (top-level step %2%, %3% new steps, %4% new runnable steps)")

[3.3558]

[11.361]

        printMsg(lvlChatty, format("added build %1% (top-level step %2%, %3% new steps, %4% new runnable steps)")

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 630 [4.4840]

B:BD[4.11654] → [4.11654:11731]

    printMsg(lvlInfo, format("considering derivation ‘%1%’") % drvPath);

[4.11654]

[4.11731]

    printMsg(lvlDebug, format("considering derivation ‘%1%’") % drvPath);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 655 [4.4840]

B:BD[4.12166] → [4.12166:12240]

    printMsg(lvlInfo, format("creating build step ‘%1%’") % drvPath);

[4.12166]

[4.12240]

    printMsg(lvlDebug, format("creating build step ‘%1%’") % drvPath);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 687 [4.4840]

∅:D[5.10174] → [3.5073:5155]

B:BD[4.12643] → [3.5073:5155]

    printMsg(lvlInfo, format("destroying build step ‘%1%’") % step->drvPath);

[5.10174]

[3.5155]

    printMsg(lvlDebug, format("destroying build step ‘%1%’") % step->drvPath);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 772 [4.4840]

B:BD[4.14048] → [5.10175:10256]

    printMsg(lvlInfo, format("step ‘%1%’ is now runnable") % step->drvPath);

[4.14048]

[5.10256]

    printMsg(lvlChatty, format("step ‘%1%’ is now runnable") % step->drvPath);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 791 [4.4840]
∅:D[12.39] → [5.10336:10387]
B:BD[5.10336] → [5.10336:10387]
```
        printMsg(lvlError, "dispatcher woken up");
```
[12.39]
[9.2577]
```
        printMsg(lvlDebug, "dispatcher woken up");
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 795 [4.4840]

B:BD[3.6839] → [5.10388:10471]

            printMsg(lvlError, format("%1% runnable builds") % runnable_->size());

[3.6839]

[5.10471]

            printMsg(lvlDebug, format("%1% runnable builds") % runnable_->size());

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 811 [4.4840]

B:BD[5.10957] → [5.10957:11064]

                    printMsg(lvlError, format("cannot execute step ‘%1%’ right now") % step->drvPath);

[5.10957]

[5.11064]

                    printMsg(lvlDebug, format("cannot execute step ‘%1%’ right now") % step->drvPath);

Deletion in src/hydra-queue-runner/hydra-queue-runner.cc at line 816 [4.4840]

B:BD[5.11138] → [5.11138:11305]

                printMsg(lvlInfo, format("WOOHOO: starting step ‘%1%’ on machine ‘%2%’")
                    % step->drvPath % reservation->machine->sshName);

Deletion in src/hydra-queue-runner/hydra-queue-runner.cc at line 877 [4.4840]
B:BD[5.13097] → [5.13097:13139]
```
    printMsg(lvlError, "builder exits");
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 912 [4.4840]

B:BD[4.15294] → [3.8006:8135]

        printMsg(lvlInfo, format("performing build step ‘%1%’ (needed by %2% builds)") % step->drvPath % dependents.size());

[4.15294]

[3.8135]

        printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by %3% builds)")
            % step->drvPath % machine->sshName % dependents.size());

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 947 [4.4840]
B:BD[13.1298] → [13.1298:1319]
```
            abort();
```
[13.1298]
[13.1319]
```
            abort(); // FIXME
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1055 [4.4840]

B:BD[4.17826] → [5.14605:14683]

    printMsg(lvlError, format("marking build %1% as succeeded") % build->id);

[4.17826]

[5.14683]

    printMsg(lvlInfo, format("marking build %1% as succeeded") % build->id);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1110 [4.4840]

B:BD[8.1131] → [12.93:131]

∅:D[12.131] → [14.812:887]

B:BD[14.812] → [14.812:887]

    printMsg(lvlError, "exiting...");
    printMsg(lvlError, format("psql connections = %1%") % dbPool.count());

[8.1131]

[4.19072]

    //printMsg(lvlInfo, "exiting...");
    //printMsg(lvlInfo, format("psql connections = %1%") % dbPool.count());