Added the InfluxDBNotification plugin including a NixOS test

[?]

Jul 25, 2019, 7:43 AM

ANJBFPBEWBYCPZULWR7MEX2BUKB4MRSJNJCZPI7G54AQYCNWKOSAC

Dependencies

[2] YWM3WYJW test.api: use Hydra perl dependencies to run `api-test.pl`
[*] T4LLYESZ * Nix expression for building Hydra.
[*] WQ2VQ7H3 Use hydra-module.nix in the tests
[*] LZVO64YG Merge in the first bits of the API work
[*] 5EQYVRWE Add a plugin mechanism
[*] G2ZB6464 first test, not yet in buildprocess

Change contents

edit in release.nix at line 31

[5.525]

[5.525]

# The following is to work around the following error from hydra-server:
# [error] Caught exception in engine "Cannot determine local time zone"
time.timeZone = "UTC";

nix = {
# The following is to work around: https://github.com/NixOS/hydra/pull/432
buildMachines = [
{ hostName = "localhost";
system = "x86_64-linux";
}
];
# Without this nix tries to fetch packages from the default
# cache.nixos.org which is not reachable from this sandboxed NixOS test.
binaryCaches = [];
};
edit in release.nix at line 239

[2.135]

[6.1720]

'';
});

tests.notifications = genAttrs' (system:
with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
simpleTest {
machine = { pkgs, ... }: {
imports = [ (hydraServer build.${system}) ];
services.hydra-dev.extraConfig = ''
<influxdb>
url = http://127.0.0.1:8086
db = hydra
</influxdb>
edit in release.nix at line 253

[6.1732]

[6.1732]

services.influxdb.enable = true;
};
testScript = ''
$machine->waitForJob("hydra-init");

# Create an admin account and some other state.
$machine->succeed
( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\""
, "mkdir /run/jobset"
, "chmod 755 /run/jobset"
, "cp ${./tests/api-test.nix} /run/jobset/default.nix"
, "chmod 644 /run/jobset/default.nix"
, "chown -R hydra /run/jobset"
);

# Wait until InfluxDB can receive web requests
$machine->waitForJob("influxdb");
$machine->waitForOpenPort("8086");

# Create an InfluxDB database where hydra will write to
$machine->succeed(
"curl -XPOST 'http://127.0.0.1:8086/query' \\
--data-urlencode 'q=CREATE DATABASE hydra'");

# Wait until hydra-server can receive HTTP requests
$machine->waitForJob("hydra-server");
$machine->waitForOpenPort("3000");

# Setup the project and jobset
$machine->mustSucceed(
"su - hydra -c 'perl -I ${build.${system}.perlDeps}/lib/perl5/site_perl ${./tests/setup-notifications-jobset.pl}' >&2");

# Wait until hydra has build the job and
# the InfluxDBNotification plugin uploaded its notification to InfluxDB
$machine->waitUntilSucceeds(
"curl -s -H 'Accept: application/csv' \\
-G 'http://127.0.0.1:8086/query?db=hydra' \\
--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success");
'';
file addition: InfluxDBNotification.pm (----------)

[7.1]

package Hydra::Plugin::InfluxDBNotification;

use strict;
use parent 'Hydra::Plugin';
use HTTP::Request;
# use JSON;
use LWP::UserAgent;
# use Hydra::Helper::CatalystUtils;

sub toBuildStatusDetailed {
my ($buildStatus) = @_;
if ($buildStatus == 0) {
return "success";
}
elsif ($buildStatus == 1) {
return "failure";
}
elsif ($buildStatus == 2) {
return "dependency-failed";
}
elsif ($buildStatus == 4) {
return "cancelled";
}
elsif ($buildStatus == 6) {
return "failed-with-output";
}
elsif ($buildStatus == 7) {
return "timed-out";
}
elsif ($buildStatus == 9) {
return "unsupported-system";
}
elsif ($buildStatus == 10) {
return "log-limit-exceeded";
}
elsif ($buildStatus == 11) {
return "output-limit-exceeded";
}
elsif ($buildStatus == 12) {
return "non-deterministic-build";
}
else {
return "aborted";
}
}

sub toBuildStatusClass {
my ($buildStatus) = @_;
if ($buildStatus == 0) {
return "success";
}
elsif ($buildStatus == 3
|| $buildStatus == 4
|| $buildStatus == 8
|| $buildStatus == 10
|| $buildStatus == 11)
{
return "canceled";
}
else {
return "failed";
}
}

# Syntax
# build_status,job=my-job status=failed,result=dependency-failed duration=123i
# | -------------------- -------------- |
# | | | |
# | | | |
# +-----------+--------+-+---------+-+---------+
# |measurement|,tag_set| |field_set| |timestamp|
# +-----------+--------+-+---------+-+---------+
sub createLine {
my ($measurement, $tagSet, $fieldSet, $timestamp) = @_;
my @tags = ();
foreach my $tag (sort keys %$tagSet) {
push @tags, "$tag=$tagSet->{$tag}";
}
my @fields = ();
foreach my $field (sort keys %$fieldSet) {
push @fields, "$field=$fieldSet->{$field}";
}
my $tags = join(",", @tags);
my $fields = join(",", @fields);
return "$measurement,$tags $fields $timestamp";
}

sub buildFinished {
my ($self, $build, $dependents) = @_;
my $influxdb = $self->{config}->{influxdb};

# skip if we didn't configure
return unless defined $influxdb;
# skip if we didn't set the URL and the DB
return unless ref $influxdb eq 'HASH' and exists $influxdb->{url} and exists $influxdb->{db};

my @lines = ();
foreach my $b ($build, @{$dependents}) {
my $tagSet = {
status => toBuildStatusClass($b->buildstatus),
result => toBuildStatusDetailed($b->buildstatus),
project => $b->project->name,
jobset => $b->jobset->name,
repo => ($b->jobset->name =~ /^(.*)\.pr-/) ? $1 : $b->jobset->name,
job => $b->job->name,
system => $b->system,
cached => $b->iscachedbuild ? "true" : "false",
};
my $fieldSet = {
# this line is needed to be able to query the statuses
build_status => $b->buildstatus . "i",
build_id => '"' . $b->id . '"',
main_build_id => '"' . $build->id . '"',
duration => ($b->stoptime - $b->starttime) . "i",
queued => ($b->starttime - $b->timestamp > 0 ? $b->starttime - $b->timestamp : 0) . "i",
closure_size => ($b->closuresize // 0) . "i",
size => ($b->size // 0) . "i",
};
my $line =
createLine("hydra_build_status", $tagSet, $fieldSet, $b->stoptime);
push @lines, $line;
}

my $payload = join("\n", @lines);
print STDERR "sending InfluxDB measurements to server $influxdb->{url}:\n$payload\n";

my $ua = LWP::UserAgent->new();
my $req = HTTP::Request->new('POST',
"$influxdb->{url}/write?db=$influxdb->{db}&precision=s");
$req->header('Content-Type' => 'application/x-www-form-urlencoded');
$req->content($payload);
my $res = $ua->request($req);
print STDERR $res->status_line, ": ", $res->decoded_content, "\n"
unless $res->is_success;
}

1;
file addition: setup-notifications-jobset.pl (----------)

[8.73]

use LWP::UserAgent;
use JSON;

my $ua = LWP::UserAgent->new;
$ua->cookie_jar({});

sub request_json {
my ($opts) = @_;
my $req = HTTP::Request->new;
$req->method($opts->{method} or "GET");
$req->uri("http://localhost:3000$opts->{uri}");
$req->header(Accept => "application/json");
$req->header(Referer => "http://localhost:3000/") if $opts->{method} eq "POST";
$req->content(encode_json($opts->{data})) if defined $opts->{data};
my $res = $ua->request($req);
print $res->as_string();
return $res;
}

my $result = request_json({
uri => "/login",
method => "POST",
data => {
username => "root",
password => "foobar"
}
});

$result = request_json({
uri => '/project/sample',
method => 'PUT',
data => {
displayname => "Sample",
enabled => "1",
visible => "1",
}
});

$result = request_json({
uri => '/jobset/sample/default',
method => 'PUT',
data => {
nixexprpath => "default.nix",
nixexprinput => "my-src",
inputs => {
"my-src" => {
type => "path",
value => "/run/jobset"
}
},
enabled => "1",
visible => "1",
checkinterval => "5",
keepnr => 1
}
});