grahamc/hydra2 - Change HPJKBFZ4C24DNBFESAVVFHOBHXOPYOP4XNPFVGOLA7EPLZCIRS6QC

Handle concurrent finishing of the same build

There is a slight possibility that the queue monitor and a builder thread simultaneously decide to mark a build as finished. That's fine, as long as we ensure the DB update is idempotent (as ensured by doing "update Builds set finished = 1 … where finished = 0").

Created by Eelco Dolstra on June 18, 2015

HPJKBFZ4C24DNBFESAVVFHOBHXOPYOP4XNPFVGOLA7EPLZCIRS6QC

Dependencies

In channels

main

Change contents

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 166 [4.4840]

B:BD[3.187] → [3.187:256]

        printMsg(lvlError, format("destroying step %1%") % drvPath);

[3.187]

[3.256]

        //printMsg(lvlError, format("destroying step %1%") % drvPath);

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 523 [4.4840]

∅:D[5.271] → [6.396:430]

B:BD[7.9876] → [6.396:430]

B:BD[6.430] → [3.324:366]

∅:D[3.366] → [4.10645:10675]

∅:D[6.430] → [4.10645:10675]

∅:D[8.641] → [4.10645:10675]

B:BD[4.10645] → [4.10645:10675]

B:BD[4.10675] → [9.255:394]

∅:D[9.394] → [4.10804:10994]

B:BD[4.10804] → [4.10804:10994]

B:BD[4.10994] → [2.0:40]

∅:D[2.40] → [10.255:283]

B:BD[4.10994] → [10.255:283]

            pqxx::work txn(conn);
            assert(!build->finishedInDB);
            txn.parameterized
                ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1")
                (build->id)
                ((int) bsAborted)
                (time(0))
                ("derivation was garbage-collected prior to build").exec();
            txn.commit();
            build->finishedInDB = true;
            nrBuildsDone++;

[5.271]

[11.295]

            if (!build->finishedInDB) {
                pqxx::work txn(conn);
                txn.parameterized
                    ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
                    (build->id)
                    ((int) bsAborted)
                    (time(0))
                    ("derivation was garbage-collected prior to build").exec();
                txn.commit();
                build->finishedInDB = true;
                nrBuildsDone++;
            }

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 602 [4.4840]

B:BD[12.310] → [13.772:810]

B:BD[13.810] → [14.0:72]

B:BD[14.72] → [3.503:549]

∅:D[14.72] → [12.310:344]

∅:D[3.549] → [12.310:344]

∅:D[13.810] → [12.310:344]

B:BD[12.310] → [12.310:344]

B:BD[12.344] → [13.811:959]

∅:D[13.959] → [12.472:504]

B:BD[12.472] → [12.472:504]

B:BD[12.504] → [13.960:1093]

∅:D[15.269] → [12.653:683]

∅:D[13.1167] → [12.653:683]

B:BD[12.653] → [12.653:683]

B:BD[12.683] → [2.41:85]

∅:D[2.85] → [10.284:316]

B:BD[12.683] → [10.284:316]

                pqxx::work txn(conn);
                createBuildStep(txn, 0, build, r, "", buildStepStatus);
                assert(!build->finishedInDB);
                txn.parameterized
                    ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1")
                    (build->id)
                    ((int) buildStatus)
                    (now)
                    (buildStatus != bsUnsupported ? 1 : 0).exec();
                txn.commit();
                build->finishedInDB = true;
                nrBuildsDone++;

[12.310]

[13.1168]

                if (!build->finishedInDB) {
                    pqxx::work txn(conn);
                    createBuildStep(txn, 0, build, r, "", buildStepStatus);
                    txn.parameterized
                        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1 and finished = 0")
                        (build->id)
                        ((int) buildStatus)
                        (now)
                        (buildStatus != bsUnsupported ? 1 : 0).exec();
                    txn.commit();
                    build->finishedInDB = true;
                    nrBuildsDone++;
                }

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 627 [4.4840]

B:BD[16.3424] → [16.3424:3467]

            (*builds_)[build->id] = build;

[16.3380]

[16.3511]

            if (!build->finishedInDB) // FIXME: can this happen?
                (*builds_)[build->id] = build;

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1238 [4.4840]

B:BD[3.8625] → [3.8625:8675]

                    assert(!build->finishedInDB);

[3.8625]

[3.8675]

                    if (build->finishedInDB) continue;

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1240 [4.4840]

B:BD[3.8713] → [3.8713:8865]

                        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")

[3.8713]

[3.8865]

                        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0")

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1278 [4.4840]
B:BD[3.9916] → [3.9916:9950]
```
    assert(!build->finishedInDB);
```
[3.9916]
[7.14683]
```
    if (build->finishedInDB) return;
```

Replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1281 [4.4840]

B:BD[17.600] → [9.695:878]

        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1")

[17.600]

[4.18032]

        ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1 and finished = 0")

Insertion in src/hydra-queue-runner/sync.hh at line 31 [16.9720]
[16.10176]
[16.10176]
```
    Sync() { }
    Sync(const T & data) : data(data) { }
```