Allow determinism checking for entire jobsets

[?]

Dec 7, 2016, 2:57 PM

EPWEMRI23UN6C777AL2GWEOXYZRTPEPFC2SZVZ4JTWUGYVFDL5XAC

Dependencies

[2] J5KJ6IIC Add tooltips to metrics showing the exact value of the data point
[3] 2DNPZFPN Step cancellation: Don't use pthread_cancel()
[4] EFWDY2V7 Support testing build determinism
[5] MHVIT4JY Split hydra-queue-runner.cc more
[6] 2GUAKGTB Fix indentation of build.tt
[7] TX7Q4RAS Add page showing latest build steps
[8] 62MQPRXC Pass null values to libpqxx properly
[9] OCZ4LSGG Automatically retry aborted builds
[10] 73YR46NJ hydra-queue-runner: Write directly to a binary cache
[11] PH3DFCNU Render machine correctly if it doesn't contain @
[12] HH3LID6L Re-implement log size limits
[13] BRAESISH Warn if PostgreSQL appears stalled
[14] UQQ4IL55 Add a error type for "unsupported system type"
[15] OBOTGFG6 Prevent orphaned build steps
[16] DKJFD6JN Process Nix API changes
[17] JGLE5BRN Add separate build step status codes for cached failures and timeouts
[18] BYVRA54Q Temporarily disable machines on any exception, not just connection failures
[19] 5AIYUMTB Basic remote building
[20] FITVNQ2S Keep track of the time we spend copying to/from build machines
[21] FCTX433O Add buildStarted plugin hook
[22] OG3Z3QGC Namespace cleanup
[23] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation
[24] PMNWRTGJ Add multiple output support
[25] 7LFMSF4K Don't show "localhost" as machine for cached failed build steps
[26] 24BMQDZA Start of single-process hydra-queue-runner
[27] VQISTKOP hydra-queue-runner: Use substitutes
[28] BG6PEOB2 Make the output size limit configurable
[29] UNVLTCV4 Fix showing machine name for aborted build steps
[30] N4IROACV Move buildRemote() into State
[31] LVQXQIYA Kill active build steps when builds are cancelled
[32] NQ2X3Y4K Don't render machine name if not applicable to step
[33] UJOSXBZI Record which build a failed build step was propagated from
[34] HJOEIMLR Refactor
[35] K22TMPH5 Make the info tables less compressed
[36] OPN3PED2 Tweak
[37] UNVMKJV5 Unify build and step status codes
[38] 7LWB2J2Z Periodically clear orphaned build steps
[39] KBZHIGLG Record the machine used for a build step
[40] NTEDD7T4 Provide a plugin hook for when build steps finish
[*] V2UCCYN3 hydra-queue-runner: Get store mode configuration from hydra.conf
[*] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
[*] CLJQCY2X * Store info about all the build actions and allow them to be
[*] EFWN7JBV * Added a status page that shows all the currently executing build steps.
[*] J5UVLXOK * Start of a basic Catalyst web interface.
[*] U2X3Y53G make sure status images always display for a list of builds
[*] N22GPKYT * Put info about logs / build products in the DB.
[*] 3E6IP3R3 * Add the name of the jobset to ReleaseSetJobs, otherwise we can't
[*] D5QIOJGP * Move everything up one directory.

Change contents

replacement in src/hydra-queue-runner/build-remote.cc at line 119

[5.215]→[5.215:274](∅→∅)

unsigned int maxSilentTime, unsigned int buildTimeout,

[5.215]

[3.0]

unsigned int maxSilentTime, unsigned int buildTimeout, unsigned int repeats,
replacement in src/hydra-queue-runner/build-remote.cc at line 266

[5.3765]→[4.261:427](∅→∅)

if (GET_PROTOCOL_MINOR(remoteVersion) >= 3)
// FIXME: make the number of repeats configurable.
to << (step->isDeterministic ? 1 : 0);

[5.3765]

[5.3765]

if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
to << repeats // == build-repeat
<< step->isDeterministic; // == enforce-determinism
}
edit in src/hydra-queue-runner/build-remote.cc at line 299

[5.4660]

[5.4660]

if (GET_PROTOCOL_MINOR(remoteVersion) >= 3) {
result.timesBuilt = readInt(from);
result.isNonDeterministic = readInt(from);
}
edit in src/hydra-queue-runner/builder.cc at line 89

[5.553]

[5.2164]

unsigned int repeats = step->isDeterministic ? 1 : 0;
edit in src/hydra-queue-runner/builder.cc at line 116

[5.90]

[5.401]

}
{
auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName));
if (i != jobsetRepeats.end())
repeats = std::max(repeats, i->second);
replacement in src/hydra-queue-runner/builder.cc at line 130

[5.741]→[5.3215:3327](∅→∅),[5.3215]→[5.3215:3327](∅→∅),[5.3327]→[5.742:827](∅→∅)

printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)")
% step->drvPath % machine->sshName % buildId % (dependents.size() - 1));

[5.741]

[5.3414]

printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)",
step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1));
replacement in src/hydra-queue-runner/builder.cc at line 171

[5.4376]→[3.1061:1161](∅→∅)

buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, result, activeStep);

[5.4376]

[5.2415]

buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, repeats, result, activeStep);
replacement in src/hydra-queue-runner/builder.cc at line 233

[5.5536]→[5.1885:1979](∅→∅),[5.1979]→[5.0:79](∅→∅),[5.530]→[5.0:79](∅→∅)

finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, buildId,
stepNr, machine->sshName, result.stepStatus, result.errorMsg);

[5.5536]

[5.5687]

finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
replacement in src/hydra-queue-runner/builder.cc at line 287

[5.7356]→[5.531:620](∅→∅),[5.620]→[5.1980:2047](∅→∅)

finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
buildId, stepNr, machine->sshName, bsSuccess);

[5.7356]

[5.7478]

finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
replacement in src/hydra-queue-runner/builder.cc at line 393

[5.477]→[5.691:784](∅→∅),[5.3420]→[5.691:784](∅→∅),[5.12039]→[5.691:784](∅→∅),[5.784]→[5.2409:2505](∅→∅)

finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
buildId, stepNr, machine->sshName, result.stepStatus, result.errorMsg);

[5.477]

[5.478]

finishBuildStep(txn, result, buildId, stepNr, machine->sshName);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 267

[5.9262]→[5.881:985](∅→∅),[5.985]→[5.703:794](∅→∅),[5.777]→[5.1071:1129](∅→∅),[5.794]→[5.1071:1129](∅→∅),[5.1071]→[5.1071:1129](∅→∅)

void State::finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime, unsigned int overhead,
BuildID buildId, unsigned int stepNr, const std::string & machine, BuildStatus status,
const std::string & errorMsg, BuildID propagatedFrom)

[5.9262]

[5.9436]

void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
BuildID buildId, unsigned int stepNr, const std::string & machine)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 270

[5.9438]→[5.9555:9600](∅→∅)

assert(startTime);
assert(stopTime);

[5.9438]

[5.391]

assert(result.startTime);
assert(result.stopTime);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 273

[5.413]→[5.1130:1315](∅→∅),[5.942]→[5.414:535](∅→∅),[5.1315]→[5.414:535](∅→∅),[5.9757]→[5.414:535](∅→∅),[5.9611]→[5.414:535](∅→∅),[5.535]→[5.943:973](∅→∅)

("update BuildSteps set busy = 0, status = $1, propagatedFrom = $4, errorMsg = $5, startTime = $6, stopTime = $7, machine = $8, overhead = $9 where build = $2 and stepnr = $3")
((int) status)(buildId)(stepNr)
(propagatedFrom, propagatedFrom != 0)
(errorMsg, errorMsg != "")
(startTime)(stopTime)

[5.413]

[5.1316]

("update BuildSteps set busy = 0, status = $1, errorMsg = $4, startTime = $5, stopTime = $6, machine = $7, overhead = $8, timesBuilt = $9, isNonDeterministic = $10 where build = $2 and stepnr = $3")
((int) result.stepStatus)(buildId)(stepNr)
(result.errorMsg, result.errorMsg != "")
(result.startTime)(result.stopTime)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 278

[5.1349]→[5.1349:1391](∅→∅)

(overhead, overhead != 0).exec();

[5.1349]

[5.13421]

(result.overhead, result.overhead != 0)
(result.timesBuilt, result.timesBuilt > 0)
(result.isNonDeterministic, result.timesBuilt > 1)
.exec();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 813

[42.1250]

[43.3753]

// FIXME: hacky mechanism for configuring determinism checks.
for (auto & s : tokenizeString<Strings>(hydraConfig["xxx-jobset-repeats"])) {
auto s2 = tokenizeString<std::vector<std::string>>(s, ":");
if (s2.size() != 3) throw Error("bad value in xxx-jobset-repeats");
jobsetRepeats.emplace(std::make_pair(s2[0], s2[1]), std::stoi(s2[2]));
}
edit in src/hydra-queue-runner/state.hh at line 50

[5.4209]

[5.4209]

unsigned int timesBuilt = 0;
bool isNonDeterministic = false;
edit in src/hydra-queue-runner/state.hh at line 419

[5.1950]

[5.1950]

/* How often the build steps of a jobset should be repeated in
order to detect non-determinism. */
std::map<std::pair<std::string, std::string>, unsigned int> jobsetRepeats;
replacement in src/hydra-queue-runner/state.hh at line 447

[5.7255]→[5.1424:1502](∅→∅),[5.1502]→[5.3425:3494](∅→∅),[5.3494]→[5.1122:1214](∅→∅),[5.1562]→[5.1122:1214](∅→∅),[5.763]→[5.7453:7490](∅→∅),[5.1214]→[5.7453:7490](∅→∅),[5.7453]→[5.7453:7490](∅→∅)

void finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime,
unsigned int overhead, BuildID buildId, unsigned int stepNr,
const std::string & machine, BuildStatus status, const std::string & errorMsg = "",
BuildID propagatedFrom = 0);

[5.7255]

[5.3136]

void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr,
const std::string & machine);
edit in src/hydra-queue-runner/state.hh at line 500

[5.1984]

[3.2341]

unsigned int repeats,
edit in src/lib/Hydra/Schema/BuildSteps.pm at line 104

[5.1621]

[45.722]

is_nullable: 1

=head2 timesbuilt

data_type: 'integer'
is_nullable: 1

=head2 isnondeterministic

data_type: 'boolean'
edit in src/lib/Hydra/Schema/BuildSteps.pm at line 145

[5.1684]

[44.2685]

"timesbuilt",
{ data_type => "integer", is_nullable => 1 },
"isnondeterministic",
{ data_type => "boolean", is_nullable => 1 },
replacement in src/lib/Hydra/Schema/BuildSteps.pm at line 218

[5.499]→[5.1685:1827](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2016-02-16 18:04:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TRALbEoaF/OIOyERYCyxkw

[5.499]

[5.10118]

# Created by DBIx::Class::Schema::Loader v0.07045 @ 2016-12-07 13:48:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3FYkqSUfgWmiqZzmX8J4TA
replacement in src/root/build.tt at line 53

[5.69]→[5.985:1002](∅→∅),[5.200]→[5.985:1002](∅→∅),[5.240]→[5.985:1002](∅→∅),[5.240]→[5.985:1002](∅→∅),[5.958]→[5.985:1002](∅→∅),[5.5497]→[5.985:1002](∅→∅),[5.985]→[5.985:1002](∅→∅)

<td>

[5.5497]

[5.1002]

<td class="step-status">
replacement in src/root/build.tt at line 57

[5.1125]→[5.1125:1151](∅→∅)

Succeeded

[5.1125]

[5.1215]

[% IF step.isnondeterministic %]
Succeeded with non-determistic result
[% ELSE %]
Succeeded
[% END %]
[% IF step.timesbuilt > 1 %]
([% step.timesbuilt %] times)
[% END %]
replacement in src/root/build.tt at line 66

[5.1258]→[5.6782:6911](∅→∅),[5.1194]→[5.6782:6911](∅→∅)

Aborted[% IF step.errormsg %]: [% HTML.escape(step.errormsg); END %]

[5.1258]

[5.5498]

Aborted[% IF step.errormsg %]: [% HTML.escape(step.errormsg) %][% END %]
replacement in src/root/build.tt at line 80

[4.836]→[4.836:903](∅→∅)

Non-deterministic build

[4.836]

[5.20001]

Non-determinism detected [% IF step.timesbuilt %] after [% step.timesbuilt %] times[% END %]
replacement in src/root/build.tt at line 82

[5.20041]→[5.20041:20125](∅→∅)

Failed: [% HTML.escape(step.errormsg) %]

[5.20041]

[5.1245]

Failed: [% HTML.escape(step.errormsg) %]
replacement in src/root/build.tt at line 147

[5.2323]→[5.2323:2391](∅→∅)

[% INCLUDE renderBuildStatusIcon size=128, build=build %]

[5.2323]

[5.2391]

[% INCLUDE renderBuildStatusIcon size=128 build=build %]
edit in src/root/static/css/hydra.css at line 134

[2.230]

[47.1098]

}

td.step-status span.error {
color: red;
font-weight: bold;
edit in src/root/static/css/hydra.css at line 140

[47.1100]

td.step-status span.warn {
color: #aaaa00;
font-weight: bold;
}
edit in src/sql/hydra.sql at line 291

[5.1927]

[5.1927]

-- How many times this build step was done (for checking determinism).
timesBuilt integer,
edit in src/sql/hydra.sql at line 295

[5.1928]

[49.5891]

-- Whether this build step produced different results when repeated.
isNonDeterministic boolean,
file addition: upgrade-52.sql (----------)

[50.3004]

alter table BuildSteps
add column timesBuilt integer,
add column isNonDeterministic boolean;