Add command ‘hydra-queue-runner --status’ to show current status

[?]

Jun 22, 2015, 12:06 PM

PLOZBRTR6USSGJX7GR2RZKNPVYG2Q6QM7LW6IA35MKL63ZTQVD7QC

Dependencies

[2] JFD25IUU hydra-queue-runner: Implement --unlock
[3] NNOCZ4RO hydra-queue-runner: Improve dispatcher
[4] GS4BE6TB Asynchronously compress build logs
[5] 7LB6QBXY Keep track of the number of build steps that are being built
[6] 5AIYUMTB Basic remote building
[7] FQQRJUO4 Mark builds as busy
[8] IWB3F4Z6 Fail builds with previously failed steps early
[9] RQUAATWB Add status dump facility
[10] OCZ4LSGG Automatically retry aborted builds
[11] WKJFPR77 hydra-queue-runner: Maintain count of active build steps
[12] 24BMQDZA Start of single-process hydra-queue-runner
[13] YZAI5GQU Implement a database connection pool
[14] NJJ7H64S Very basic multi-threaded queue runner
[15] 62MQPRXC Pass null values to libpqxx properly
[16] T2EIYJNG On SIGINT, shut down the builder threads
[17] GKZN4UV7 Make the queue monitor more robust, and better debug output
[18] PQFOMNTL hydra-queue-runner: More stats
[19] XV4AEKJC hydra-queue-runner: Handle status queries on the main thread
[*] N22GPKYT * Put info about logs / build products in the DB.
[*] RYTQLATY Keep track of failed paths in the Hydra database
[*] KSBB33RE Add a dashboard
[*] D5QIOJGP * Move everything up one directory.

Change contents

edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 28

[6.5036]

[6.5036]

#include "value-to-json.hh"
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 259

[6.6441]→[6.6441:6456](∅→∅)

~State();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 260

[6.6457]→[4.853:890](∅→∅)

void clearBusy(time_t stopTime);

[6.6457]

[6.7280]

private:
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 262

[6.7281]→[4.891:900](∅→∅)

private:

[6.7281]

[4.900]

void clearBusy(Connection & conn, time_t stopTime);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 309

[4.1020]

[4.1020]

void dumpStatus(Connection & conn);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 313

[4.1029]

[4.1029]

void showStatus();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 316

[4.1030]→[6.66:89](∅→∅),[6.66]→[6.66:89](∅→∅)

void dumpStatus();

[4.1030]

[6.1377]

void unlock();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 331

[6.7507]→[6.7507:7535](∅→∅),[6.7535]→[6.167:237](∅→∅),[6.237]→[6.109:137](∅→∅),[6.109]→[6.109:137](∅→∅),[6.137]→[6.7676:7731](∅→∅),[6.330]→[6.7676:7731](∅→∅),[6.7676]→[6.7676:7731](∅→∅),[6.7731]→[6.7893:7895](∅→∅)

State::~State()
{
try {
printMsg(lvlInfo, "clearing active builds / build steps...");
clearBusy(time(0));
} catch (...) {
ignoreException();
}
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 376

[6.9443]→[6.138:177](∅→∅)

void State::clearBusy(time_t stopTime)

[6.9443]

[6.7817]

void State::clearBusy(Connection & conn, time_t stopTime)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 378

[6.7819]→[6.391:447](∅→∅)

auto conn(dbPool.get());
pqxx::work txn(*conn);

[6.7819]

[6.0]

pqxx::work txn(conn);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1399

[4.3009]→[6.256:281](∅→∅),[6.3180]→[6.256:281](∅→∅)

void State::dumpStatus()

[4.3009]

[6.281]

void State::dumpStatus(Connection & conn)
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1401

[6.283]

[6.283]

std::ostringstream out;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1404

[6.289]→[6.289:401](∅→∅)

auto builds_(builds.lock());
printMsg(lvlError, format("%1% queued builds") % builds_->size());

[6.289]

[6.401]

JSONObject root(out);
root.attr("status", "up");
root.attr("time", time(0));
root.attr("pid", getpid());
{
auto builds_(builds.lock());
root.attr("nrQueuedBuilds", builds_->size());
}
{
auto steps_(steps.lock());
for (auto i = steps_->begin(); i != steps_->end(); )
if (i->second.lock()) ++i; else i = steps_->erase(i);
root.attr("nrUnfinishedSteps", steps_->size());
}
{
auto runnable_(runnable.lock());
for (auto i = runnable_->begin(); i != runnable_->end(); )
if (i->lock()) ++i; else i = runnable_->erase(i);
root.attr("nrRunnableSteps", runnable_->size());
}
root.attr("nrActiveSteps", nrActiveSteps);
root.attr("nrStepsBuilding", nrStepsBuilding);
root.attr("nrBuildsRead", nrBuildsRead);
root.attr("nrBuildsDone", nrBuildsDone);
root.attr("nrStepsDone", nrStepsDone);
root.attr("nrRetries", nrRetries);
root.attr("maxNrRetries", maxNrRetries);
root.attr("nrQueueWakeups", nrQueueWakeups);
root.attr("nrDispatcherWakeups", nrDispatcherWakeups);
root.attr("nrDbConnections", dbPool.count());
{
root.attr("machines");
JSONObject nested(out);
auto machines_(machines.lock());
for (auto & m : *machines_) {
nested.attr(m->sshName);
JSONObject nested2(out);
nested2.attr("currentJobs", m->currentJobs);
nested2.attr("maxJobs", m->maxJobs);
}
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1446

[6.407]

[6.407]
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1448

[6.413]→[6.413:662](∅→∅)

auto steps_(steps.lock());
for (auto i = steps_->begin(); i != steps_->end(); )
if (i->second.lock()) ++i; else i = steps_->erase(i);
printMsg(lvlError, format("%1% pending/active build steps") % steps_->size());

[6.413]

[6.662]

pqxx::work txn(conn);
// FIXME: use PostgreSQL 9.5 upsert.
txn.exec("delete from SystemStatus where what = 'queue-runner'");
txn.parameterized("insert into SystemStatus values ('queue-runner', $1)")(out.str()).exec();
txn.exec("notify status_dumped");
txn.commit();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1455

[6.668]

[6.668]

}

void State::showStatus()
{
auto conn(dbPool.get());
receiver statusDumped(*conn, "status_dumped");

string status;
bool barf = false;

/* Get the last JSON status dump from the database. */
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1468

[6.674]→[6.674:928](∅→∅)

auto runnable_(runnable.lock());
for (auto i = runnable_->begin(); i != runnable_->end(); )
if (i->lock()) ++i; else i = runnable_->erase(i);
printMsg(lvlError, format("%1% runnable build steps") % runnable_->size());

[6.674]

[6.928]

pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
}

if (status != "") {

/* If the status is not empty, then the queue runner is
running. Ask it to update the status dump. */
{
pqxx::work txn(*conn);
txn.exec("notify dump_status");
txn.commit();
}

/* Wait until it has done so. */
barf = conn->await_notification(5, 0) == 0;

/* Get the new status. */
{
pqxx::work txn(*conn);
auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'");
if (res.size()) status = res[0][0].as<string>();
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1494

[6.934]→[6.277:351](∅→∅),[6.351]→[5.618:706](∅→∅),[6.351]→[6.431:644](∅→∅),[5.706]→[6.431:644](∅→∅),[6.934]→[6.431:644](∅→∅),[6.644]→[6.6398:6781](∅→∅),[6.934]→[6.6398:6781](∅→∅)

printMsg(lvlError, format("%1% active build steps") % nrActiveSteps);
printMsg(lvlError, format("%1% build steps currently building") % nrStepsBuilding);
printMsg(lvlError, format("%1% builds read from queue") % nrBuildsRead);
printMsg(lvlError, format("%1% builds done") % nrBuildsDone);
printMsg(lvlError, format("%1% build steps done") % nrStepsDone);
printMsg(lvlError, format("%1% build step retries") % nrRetries);
printMsg(lvlError, format("%1% most retries for any build step") % maxNrRetries);
printMsg(lvlError, format("%1% queue wakeups") % nrQueueWakeups);
printMsg(lvlError, format("%1% dispatcher wakeups") % nrDispatcherWakeups);
printMsg(lvlError, format("%1% database connections") % dbPool.count());

[6.934]

[6.1024]

if (status == "") status = R"({"status":"down"})";

std::cout << status << "\n";

if (barf)
throw Error("queue runner did not respond; status information may be wrong");
}

void State::unlock()
{
auto conn(dbPool.get());

clearBusy(*conn, 0);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1511

[6.1030]→[6.1030:1109](∅→∅),[6.1163]→[6.1163:1232](∅→∅),[6.1232]→[3.3663:3724](∅→∅),[3.3724]→[6.1292:1302](∅→∅),[6.1292]→[6.1292:1302](∅→∅)

auto machines_(machines.lock());
for (auto & m : *machines_) {
printMsg(lvlError, format("machine %1%: %2%/%3% active")
% m->sshName % m->currentJobs % m->maxJobs);
}

[6.1030]

[6.1302]

pqxx::work txn(*conn);
txn.exec("delete from SystemStatus where what = 'queue-runner'");
txn.commit();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1520

[6.3200]→[6.879:897](∅→∅)

clearBusy(0);

[6.3200]

[6.3285]

{
auto conn(dbPool.get());
clearBusy(*conn, 0);
dumpStatus(*conn);
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1543

[6.633]→[6.633:674](∅→∅)

State::dumpStatus();

[6.633]

[6.674]

State::dumpStatus(*conn);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1566

[2.30]

[6.19176]

bool status = false;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1571

[2.193]

[2.193]

else if (*arg == "--status")
status = true;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1585

[6.19444]→[2.279:331](∅→∅)

if (unlock)
state.clearBusy(0);

[6.19444]

[2.331]

if (status)
state.showStatus();
else if (unlock)
state.unlock();
edit in src/sql/hydra.sql at line 532

[22.4436]

[23.8501]

create table SystemStatus (
what text primary key not null,
status json not null
);
file addition: upgrade-34.sql (----------)

[24.3004]

create table SystemStatus (
what text primary key not null,
status json not null
);