Handle concurrent finishing of the same build

[?]

Jun 18, 2015, 3:12 PM

HPJKBFZ4C24DNBFESAVVFHOBHXOPYOP4XNPFVGOLA7EPLZCIRS6QC

Dependencies

[2] X7AZHNKG Set finishedInDB in a few more places
[3] HUUZFPPK Fix race between the queue monitor and the builder threads
[4] GKZN4UV7 Make the queue monitor more robust, and better debug output
[5] YZAI5GQU Implement a database connection pool
[6] 62MQPRXC Pass null values to libpqxx properly
[7] IWB3F4Z6 Fail builds with previously failed steps early
[8] OCZ4LSGG Automatically retry aborted builds
[9] ENXUSMSV Make concurrency more robust
[10] JAUB2FT5 getQueuedBuilds(): Handle dependent builds first
[11] PQFOMNTL hydra-queue-runner: More stats
[12] N5O7VEEO Immediately abort builds that require an unsupported system type
[13] 24BMQDZA Start of single-process hydra-queue-runner
[14] FQQRJUO4 Mark builds as busy
[15] UQQ4IL55 Add a error type for "unsupported system type"
[16] MSIHMO45 Tweak build steps
[17] 2IQRXLWE Support cancelling builds
[18] RYTQLATY Keep track of failed paths in the Hydra database
[19] UPN2SAML Acquire exclusive table lock earlier
[20] 5AIYUMTB Basic remote building

Change contents

replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 166

[3.187]→[3.187:256](∅→∅)

printMsg(lvlError, format("destroying step %1%") % drvPath);

[3.187]

[3.256]

//printMsg(lvlError, format("destroying step %1%") % drvPath);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 523

[4.271]→[4.396:430](∅→∅),[4.9876]→[4.396:430](∅→∅),[4.430]→[3.324:366](∅→∅),[3.366]→[4.10645:10675](∅→∅),[4.430]→[4.10645:10675](∅→∅),[4.641]→[4.10645:10675](∅→∅),[4.10645]→[4.10645:10675](∅→∅),[4.10675]→[4.255:394](∅→∅),[4.394]→[4.10804:10994](∅→∅),[4.10804]→[4.10804:10994](∅→∅),[4.10994]→[2.0:40](∅→∅),[2.40]→[4.255:283](∅→∅),[4.10994]→[4.255:283](∅→∅)

pqxx::work txn(conn);
assert(!build->finishedInDB);
txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1")
(build->id)
((int) bsAborted)
(time(0))
("derivation was garbage-collected prior to build").exec();
txn.commit();
build->finishedInDB = true;
nrBuildsDone++;

[4.271]

[4.295]

if (!build->finishedInDB) {
pqxx::work txn(conn);
txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
(build->id)
((int) bsAborted)
(time(0))
("derivation was garbage-collected prior to build").exec();
txn.commit();
build->finishedInDB = true;
nrBuildsDone++;
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 602

[4.310]→[4.772:810](∅→∅),[4.810]→[4.0:72](∅→∅),[4.72]→[3.503:549](∅→∅),[4.72]→[4.310:344](∅→∅),[3.549]→[4.310:344](∅→∅),[4.810]→[4.310:344](∅→∅),[4.310]→[4.310:344](∅→∅),[4.344]→[4.811:959](∅→∅),[4.959]→[4.472:504](∅→∅),[4.472]→[4.472:504](∅→∅),[4.504]→[4.960:1093](∅→∅),[4.269]→[4.653:683](∅→∅),[4.1167]→[4.653:683](∅→∅),[4.653]→[4.653:683](∅→∅),[4.683]→[2.41:85](∅→∅),[2.85]→[4.284:316](∅→∅),[4.683]→[4.284:316](∅→∅)

pqxx::work txn(conn);
createBuildStep(txn, 0, build, r, "", buildStepStatus);
assert(!build->finishedInDB);
txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1")
(build->id)
((int) buildStatus)
(now)
(buildStatus != bsUnsupported ? 1 : 0).exec();
txn.commit();
build->finishedInDB = true;
nrBuildsDone++;

[4.310]

[4.1168]

if (!build->finishedInDB) {
pqxx::work txn(conn);
createBuildStep(txn, 0, build, r, "", buildStepStatus);
txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1 and finished = 0")
(build->id)
((int) buildStatus)
(now)
(buildStatus != bsUnsupported ? 1 : 0).exec();
txn.commit();
build->finishedInDB = true;
nrBuildsDone++;
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 627

[4.3424]→[4.3424:3467](∅→∅)

(*builds_)[build->id] = build;

[4.3380]

[4.3511]

if (!build->finishedInDB) // FIXME: can this happen?
(*builds_)[build->id] = build;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1238

[3.8625]→[3.8625:8675](∅→∅)

assert(!build->finishedInDB);

[3.8625]

[3.8675]

if (build->finishedInDB) continue;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1240

[3.8713]→[3.8713:8865](∅→∅)

("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")

[3.8713]

[3.8865]

("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1278

[3.9916]→[3.9916:9950](∅→∅)

assert(!build->finishedInDB);

[3.9916]

[4.14683]

if (build->finishedInDB) return;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1281

[4.600]→[4.695:878](∅→∅)

("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1")

[4.600]

[4.18032]

("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1 and finished = 0")
edit in src/hydra-queue-runner/sync.hh at line 31

[4.10176]

[4.10176]

Sync() { }
Sync(const T & data) : data(data) { }