hydra-evaluator improvements

[?]

Mar 13, 2017, 3:19 PM

W4G5MZZSCK7VGSZJQKLS6XMQPFDIL7D542XWDMFMO4RZEO4FQB2QC

Dependencies

[2] 7XOH2DGH hydra-module.nix: Give types to options
[3] CCIORPJ5 When manually scheduling an eval, force re-instantiation of store derivations
[4] ZM6WYVL7 hydra-evaluator: Set jobset error if hydra-eval-jobset dies
[5] OPQLMRPU hydra-evaluator: Handle SIGINT
[6] W5OAZWPD Drop the errorMsg column in the Jobs table
[7] PCD3ZH6Z Partially revert 1c20cfdf2403feb78cef515faf15c04d5c9f17bd
[8] NS7SND6R hydra-evaluator: Send statistics to statsd
[9] PYTVXHMH hydra-evaluator: Show friendlier exit status
[10] Y6AHH4TH Remove the logfile and logSize columns from the database
[11] KQAQ4FIF Update Schema classes
[12] 7ZYBTNJJ revert unmeant previous change which caused errors to stay, even after successful evaluation of jobs
[13] LZVO64YG Merge in the first bits of the API work
[14] 6QRHXIM3 * Speed up the jobset index page. Especially the query to get the
[15] K3HODXGH Check all inputs for blame but only email selected inputs
[16] PCKLFRT5 Support push notification of repository changes
[17] WVD3YYON hydra-evaluator: Add some debug code
[18] NI5BVF2V * In job inputs of type "build", allow the project and jobset names of
[19] QLOLZHRX Allow a per-jobset check interval
[20] KSBB33RE Add a dashboard
[21] TFK74CQR Put job status on a separate tab
[22] OVR2RWBI hydra-evaluator: Always pick the jobset that hasn't been evaluated longest
[23] DH3KNBAV Merge remote-tracking branch 'upstream/who-broke-builds' into upstream-master
[24] M4TBFHHJ "limit" -> "rows"
[25] MPFSVI5X Pedantry: CLOCK_REALTIME -> CLOCK_MONOTONIC
[26] QTC3SYBM Jobset page: Load the jobs and status tabs on demand
[27] 4YCF3KBG Concurrent hydra-evaluator
[28] SS4TZXNU Distinguish between permanent evaluation errors and transient input errors
[29] WQXF2T3D hydra-evaluator: Don't require $HYDRA_CONFIG
[30] T5BIOVJE Add support for tracking custom metrics
[31] 5MVWRWHT hydra-evaluator: Don't use the "timeout" command
[32] X27GNHDV * Basic job info in the database.
[33] 4UUKWXDF Jobset page: Show last evaluation time
[34] 6LETVKRA After editing a jobset, push it to the front of the evaluation queue
[35] 53PW36WS hydra-evaluator: When evaluating a single jobset, exit with a failure code if evaluation fails
[36] J74UTA3I Handle the case where a jobset has never been evaluated
[37] SB2V735V Keep track of the database schema version
[38] RXVJFQ5A Evaluator cleanups
[39] GEADFVZ5 hydra-queue-runner: Improved scheduling
[*] D3MDJONY
[*] KMPLMG7T hydra-module.nix: Cleanup
[*] 3HZY24CX * Make jobsets viewable under
[*] ODNCGFQ5 * Improved the navigation bar: don't include all projects (since that
[*] N22GPKYT * Put info about logs / build products in the DB.
[*] LLAJKVGI Fix syntax error
[*] D5QIOJGP * Move everything up one directory.

Change contents

edit in hydra-module.nix at line 348

[2.1166]

[42.388]

ExecStopPost = "${cfg.package}/bin/hydra-evaluator --unlock";
edit in src/hydra-evaluator/hydra-evaluator.cc at line 8

[6.692]

[6.692]

#include <experimental/optional>
edit in src/hydra-evaluator/hydra-evaluator.cc at line 15

[6.761]

[6.761]

typedef std::pair<std::string, std::string> JobsetName;
edit in src/hydra-evaluator/hydra-evaluator.cc at line 21

[6.815]→[6.815:876](∅→∅)

typedef std::pair<std::string, std::string> JobsetName;
edit in src/hydra-evaluator/hydra-evaluator.cc at line 31

[6.1102]

[6.1102]

std::experimental::optional<JobsetName> evalOne;
edit in src/hydra-evaluator/hydra-evaluator.cc at line 64

[6.1941]

[6.1941]

if (evalOne && name != *evalOne) continue;
edit in src/hydra-evaluator/hydra-evaluator.cc at line 77

[6.2327]

[6.2327]

if (evalOne && seen.empty()) {
printError("the specified jobset does not exist");
std::_Exit(1);
}
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 93

[6.2696]→[6.2696:2799](∅→∅)

printInfo("starting evaluation of jobset ‘%s:%s’", jobset.name.first, jobset.name.second);

[6.2696]

[6.2799]

time_t now = time(0);

printInfo("starting evaluation of jobset ‘%s:%s’ (last checked %d s ago)",
jobset.name.first, jobset.name.second,
now - jobset.lastCheckedTime);

{
auto conn(dbPool.get());
pqxx::work txn(*conn);
txn.parameterized
("update Jobsets set startTime = $1 where project = $2 and name = $3")
(now)
(jobset.name.first)
(jobset.name.second)
.exec();
txn.commit();
}
edit in src/hydra-evaluator/hydra-evaluator.cc at line 122

[6.3249]→[6.3249:3752](∅→∅)

time_t now = time(0);

{
auto conn(dbPool.get());
pqxx::work txn(*conn);
txn.parameterized
("update Jobsets set lastCheckedTime = $1, triggerTime = null where project = $2 and name = $3")
(now)
(jobset.name.first)
(jobset.name.second)
.exec();
txn.commit();

jobset.lastCheckedTime = now;
jobset.triggerTime = notTriggered;
}
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 133

[6.4062]→[6.4062:4295](∅→∅)

if (i->second.pid == -1 &&
(i->second.triggerTime != std::numeric_limits<time_t>::max() ||
(i->second.checkInterval > 0 && i->second.lastCheckedTime + i->second.checkInterval <= now)))

[6.4062]

[6.4295]

if (evalOne ||
(i->second.pid == -1 &&
(i->second.triggerTime != std::numeric_limits<time_t>::max() ||
(i->second.checkInterval > 0 && i->second.lastCheckedTime + i->second.checkInterval <= now))))
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 239

[6.7713]→[6.7713:7818](∅→∅)

for (auto & jobset : state->jobsets)
if (jobset.second.pid == pid) {

[6.7713]

[6.0]

// FIXME: should use a map.
for (auto & i : state->jobsets) {
auto & jobset(i.second);

if (jobset.pid == pid) {
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 246

[6.73]→[6.73:167](∅→∅)

jobset.first.first, jobset.first.second, statusToString(status));

[6.73]

[4.0]

jobset.name.first, jobset.name.second, statusToString(status));

auto now = time(0);

jobset.triggerTime = notTriggered;
jobset.lastCheckedTime = now;
edit in src/hydra-evaluator/hydra-evaluator.cc at line 254

[4.31]

[4.31]

auto conn(dbPool.get());
pqxx::work txn(*conn);

/* Clear the trigger time to prevent this
jobset from getting stuck in an endless
failing eval loop. */
txn.parameterized
("update Jobsets set triggerTime = null where project = $1 and name = $2 and startTime is not null and triggerTime < startTime")
(jobset.name.first)
(jobset.name.second)
.exec();

/* Clear the start time. */
txn.parameterized
("update Jobsets set startTime = null where project = $1 and name = $2")
(jobset.name.first)
(jobset.name.second)
.exec();
edit in src/hydra-evaluator/hydra-evaluator.cc at line 275

[4.113]→[4.113:225](∅→∅)

auto conn(dbPool.get());
pqxx::work txn(*conn);
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 276

[4.275]→[4.275:364](∅→∅)

("update Jobsets set errorMsg = $1, errorTime = $2")

[4.275]

[4.364]

("update Jobsets set errorMsg = $1, lastCheckedTime = $2, errorTime = $2, fetchErrorMsg = null where project = $3 and name = $4")
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 278

[4.447]→[4.447:493](∅→∅)

(time(0))

[4.447]

[4.493]

(now)
(jobset.name.first)
(jobset.name.second)
edit in src/hydra-evaluator/hydra-evaluator.cc at line 282

[4.538]→[4.538:584](∅→∅)

txn.commit();
edit in src/hydra-evaluator/hydra-evaluator.cc at line 284

[4.615]

[4.615]

txn.commit();
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 290

[6.167]→[6.7990:8043](∅→∅),[4.785]→[6.7990:8043](∅→∅),[6.7990]→[6.7990:8043](∅→∅)

jobset.second.pid.release();

[4.785]

[6.8043]

jobset.pid.release();
edit in src/hydra-evaluator/hydra-evaluator.cc at line 292

[6.8093]

[6.8093]

if (evalOne) std::_Exit(0);
edit in src/hydra-evaluator/hydra-evaluator.cc at line 297

[6.8146]

[6.8146]

}
edit in src/hydra-evaluator/hydra-evaluator.cc at line 302

[6.8177]

[6.8177]

void unlock()
{
auto conn(dbPool.get());
pqxx::work txn(*conn);
txn.parameterized("update Jobsets set startTime = null").exec();
txn.commit();
}
edit in src/hydra-evaluator/hydra-evaluator.cc at line 312

[6.8198]

[5.0]

unlock();
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 315

[5.63]→[5.63:138](∅→∅)

auto callback = createInterruptCallback([&]() { std::_Exit(0); });

[5.63]

[5.138]

auto callback = createInterruptCallback([&]() { std::_Exit(1); });
edit in src/hydra-evaluator/hydra-evaluator.cc at line 341

[6.8759]

[6.8759]

bool unlock = false;

Evaluator evaluator;

std::vector<std::string> args;
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 348

[6.8854]→[6.8854:8880](∅→∅)

return false;

[6.8854]

[6.8880]

if (*arg == "--unlock")
unlock = true;
else if (hasPrefix(*arg, "-"))
return false;
args.push_back(*arg);
return true;
replacement in src/hydra-evaluator/hydra-evaluator.cc at line 356

[6.8893]→[6.8893:8920](∅→∅)

Evaluator().run();

[6.8893]

[6.8920]

if (!args.empty()) {
if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [<project> <jobset>]");
evaluator.evalOne = JobsetName(args[0], args[1]);
}

if (unlock)
evaluator.unlock();
else
evaluator.run();
replacement in src/lib/Hydra/Controller/Jobset.pm at line 46

[6.174]→[6.0:124](∅→∅)

$c->stash->{latestEval} = $c->stash->{jobset}->jobsetevals->search({}, { rows => 1, order_by => ["id desc"] })->single;

[6.174]

[6.124]

$c->stash->{latestEval} = $c->stash->{jobset}->jobsetevals->search({ hasnewbuilds => 1 }, { rows => 1, order_by => ["id desc"] })->single;
edit in src/lib/Hydra/Schema/Jobsets.pm at line 135

[3.377]

[6.131]

is_nullable: 1

=head2 starttime

data_type: 'integer'
edit in src/lib/Hydra/Schema/Jobsets.pm at line 181

[3.441]

[6.2524]

"starttime",
{ data_type => "integer", is_nullable => 1 },
replacement in src/lib/Hydra/Schema/Jobsets.pm at line 355

[6.22628]→[3.442:584](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.07045 @ 2016-10-24 20:12:51
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PSR66NnVRNTMFhDEm10erA

[6.22628]

[6.217]

# Created by DBIx::Class::Schema::Loader v0.07045 @ 2017-03-09 13:03:05
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ivYvsUyhEeaeI4EmRQ0/QQ
replacement in src/root/jobset.tt at line 97

[6.673]→[6.239:273](∅→∅)

[% IF jobset.triggertime %]

[6.673]

[6.273]

[% IF jobset.starttime %]
<tr>
<th>Evaluation running since:</th>
<td>[% INCLUDE renderRelativeDate timestamp = jobset.starttime %]</td>
</tr>
[% ELSIF jobset.triggertime %]
replacement in src/root/jobset.tt at line 105

[6.327]→[6.327:404](∅→∅)

<td>[% INCLUDE renderDateTime timestamp = jobset.triggertime %]</td>

[6.327]

[6.404]

<td>[% INCLUDE renderRelativeDate timestamp = jobset.triggertime %]</td>
edit in src/script/hydra-eval-jobset at line 752

[6.1450]→[6.214:298](∅→∅),[6.298]→[6.0:146](∅→∅),[6.146]→[6.2152:2197](∅→∅),[6.146]→[6.1451:1452](∅→∅),[6.372]→[6.1451:1452](∅→∅),[6.2197]→[6.1451:1452](∅→∅),[6.2923]→[6.1451:1452](∅→∅)

print STDERR "considering jobset ", $jobset->project->name, ":", $jobset->name,
$jobset->lastcheckedtime
? " (last checked " . (time() - $jobset->lastcheckedtime) . "s ago)\n"
: " (never checked)\n";

my $triggerTime = $jobset->triggertime;
edit in src/script/hydra-eval-jobset at line 770

[6.42]→[6.2210:2599](∅→∅),[6.2210]→[6.2210:2599](∅→∅),[6.2599]→[6.1038:1062](∅→∅)

}

if (defined $triggerTime) {
txn_do($db, sub {
# Only clear the trigger time if the jobset hasn't been
# triggered in the meantime. In that case, we need to
# evaluate again.
my $new = $jobset->get_from_storage();
$jobset->update({ triggertime => undef })
if $new->triggertime == $triggerTime;
}) if !$dryRun;
edit in src/sql/hydra.sql at line 72

[3.784]

[46.0]

startTime integer, -- if jobset is currently running
file addition: upgrade-53.sql (----------)

[47.3004]

alter table Jobsets add column startTime integer;