hydra-queue-runner: Send build notifications

[?]

Jun 22, 2015, 10:14 PM

IE2PRAQUCQVFPJ4CAIJRPXXEFC5VBAE3EO5I5FG4XWEDRNONNHKQC

Dependencies

[2] 3YHNO5H2 Don't use Perl's -w flag
[3] HUUZFPPK Fix race between the queue monitor and the builder threads
[4] K5G5GZY7 Guard against concurrent invocations of hydra-queue-runner
[5] XV4AEKJC hydra-queue-runner: Handle status queries on the main thread
[6] NJJ7H64S Very basic multi-threaded queue runner
[7] ERNOO5ZZ * Reorganising.
[8] T2EIYJNG On SIGINT, shut down the builder threads
[9] 24BMQDZA Start of single-process hydra-queue-runner
[10] HHOMBU7G hydra-queue-runner: Implement timeouts
[11] T4LLYESZ * Nix expression for building Hydra.
[12] Q6SOGMDQ Hydra/28: Rename "scheduler" to "evaluator"
[13] FV2M6MOT hydra: use autoconf/-make
[14] ENXUSMSV Make concurrency more robust
[15] 5AIYUMTB Basic remote building
[16] L4AI5YL6 Rename hydra_*.pl to hydra-*
[17] N22GPKYT * Put info about logs / build products in the DB.
[18] OCZ4LSGG Automatically retry aborted builds
[19] GS4BE6TB Asynchronously compress build logs
[20] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
[21] FULDVXE2 Periodically dump/log status
[22] D5QIOJGP * Move everything up one directory.
[23] C6HOMHZW Don't try to handle SIGINT
[24] JK2QWPH6
[25] JAUB2FT5 getQueuedBuilds(): Handle dependent builds first
[*] LJILHOJ7 Create BuildSteps race-free
[*] XHOZT4WT Add a command `hydra-create-user' for managing user accounts
[*] Y6H7Y3OT Capture the path to `guile', when available.

Change contents

edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 263

[5.852]

[5.7253]

/* Notification sender work queue. FIXME: if hydra-queue-runner is
killed before it has finished sending notifications about a
build, then the notifications may be lost. It would be better
to mark builds with pending notification in the database. */
typedef std::pair<BuildID, std::vector<BuildID>> NotificationItem;
Sync<std::queue<NotificationItem>> notificationSenderQueue;
std::condition_variable_any notificationSenderWakeup;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 324

[5.1020]

[4.25]

/* Thread that asynchronously invokes hydra-notify to send build
notifications. */
void notificationSender();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1199

[3.5204]

[5.8953]

}

/* Send notification about this build. */
{
auto notificationSenderQueue_(notificationSenderQueue.lock());
notificationSenderQueue_->push(NotificationItem(build->id, std::vector<BuildID>()));
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1206

[5.8963]

[5.8969]

notificationSenderWakeup.notify_one();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1234

[3.5932]

[27.199]

std::vector<BuildID> dependentIDs;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1297

[3.8625]

[3.8675]

dependentIDs.push_back(build2->id);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1324

[3.9914]

[5.17548]

}

/* Send notification about this build and its dependents. */
{
auto notificationSenderQueue_(notificationSenderQueue.lock());
notificationSenderQueue_->push(NotificationItem(build->id, dependentIDs));
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1331

[5.17558]

[5.17558]

notificationSenderWakeup.notify_one();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1423

[5.2319]→[5.2319:2371](∅→∅)

throw SysError("cannot start ssh");

[5.2319]

[5.2371]

throw SysError("cannot start bzip2");
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1445

[4.170]

[4.170]

void State::notificationSender()
{
while (true) {
try {
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1451

[4.171]

[4.171]

NotificationItem item;
{
auto notificationSenderQueue_(notificationSenderQueue.lock());
while (notificationSenderQueue_->empty())
notificationSenderQueue_.wait(notificationSenderWakeup);
item = notificationSenderQueue_->front();
notificationSenderQueue_->pop();
}

printMsg(lvlChatty, format("sending notification about build %1%") % item.first);

Pid pid = startProcess([&]() {
Strings argv({"hydra-notify", "build", int2String(item.first)});
for (auto id : item.second)
argv.push_back(int2String(id));
execvp("hydra-notify", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast
throw SysError("cannot start hydra-notify");
});

int res = pid.wait(true);

if (res != 0)
throw Error(format("hydra-build returned exit code %1% notifying about build %2%")
% res % item.first);

} catch (std::exception & e) {
printMsg(lvlError, format("notification sender: %1%") % e.what());
sleep(5);
}
}
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1650

[5.3364]→[5.14706:14777](∅→∅)

auto queueMonitorThread = std::thread(&State::queueMonitor, this);

[5.3364]

[5.14777]

std::thread(&State::queueMonitor, this).detach();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1658

[5.3153]

[5.399]

/* Idem for notification sending. */
std::thread(&State::notificationSender, this).detach();

/* Monitor the database for status dump requests (e.g. from
‘hydra-queue-runner --status’). */
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1676

[5.92]→[5.771:772](∅→∅),[5.889]→[5.771:772](∅→∅),[5.14845]→[5.771:772](∅→∅),[5.771]→[5.771:772](∅→∅),[5.772]→[5.890:912](∅→∅),[5.912]→[5.3479:3510](∅→∅),[5.1129]→[5.3479:3510](∅→∅),[5.3479]→[5.3479:3510](∅→∅)

// Never reached.
queueMonitorThread.join();
edit in src/script/Makefile.am at line 12

[28.421]

[29.541]

hydra-notify \
replacement in src/script/hydra-evaluator at line 1

[5.1319]→[2.80:119](∅→∅)

#! /var/run/current-system/sw/bin/perl

[5.1319]

[5.1333]

#! /run/current-system/sw/bin/perl
file addition: hydra-notify (---r------)

[5.2543]

#! /run/current-system/sw/bin/perl

use strict;
use utf8;
use Hydra::Plugin;
use Hydra::Helper::Nix;
use Hydra::Helper::PluginHooks;

STDERR->autoflush(1);
binmode STDERR, ":encoding(utf8)";

my $config = getHydraConfig();

my $db = Hydra::Model::DB->new();

my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);

my $cmd = shift @ARGV or die "Syntax: hydra-notify build BUILD-ID [BUILD-IDs...]\n";

if ($cmd eq "build") {
my $buildId = shift @ARGV or die;
my $build = $db->resultset('Builds')->find($buildId)
or die "build $buildId does not exist\n";
my @dependents;
foreach my $id (@ARGV) {
my $dep = $db->resultset('Builds')->find($id)
or die "build $id does not exist\n";
push @dependents, $dep;
}
notifyBuildFinished(\@plugins, $build, [@dependents]);
}

else {
die "unknown action ‘$cmd’";
}