* Basic job info in the database.

[?]

Nov 4, 2008, 6:23 PM

X27GNHDV5KPZ5GSH6DCAJMNCEMZLCP7M43JWF2X3O5QWXMOX273AC

Dependencies

[2] WYN733ST * Store build duration, handle cached builds.
[3] CLXEECMF * Start putting build results in a database.
[4] UVMFS73T * Some jQuery / CSS hackery.
[5] N22GPKYT * Put info about logs / build products in the DB.
[6] J5UVLXOK * Start of a basic Catalyst web interface.
[7] ELCI5T2A * Show the latest build for each job.

Change contents

file deletion: test.nix (----------)

[3.2]→[3.242:274](∅→∅),[3.274]→[3.21:21](∅→∅)

let

pkgs = import (builtins.getEnv "NIXPKGS_ALL") {};

pkgs64 = import (builtins.getEnv "NIXPKGS_ALL") {system = "x86_64-linux";};

in

{

job1 = pkgs.hello;

job1_64 = pkgs64.hello;

job2 = pkgs.aterm;

}
replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm at line 28

[3.3992]→[2.421:563](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xXiHLBKW5fHl7ukdYeIsTw

[3.3992]

[3.4134]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kck2qlNZVLFUnevNPSBVKw
replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm at line 28

[3.4922]→[2.564:706](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5SPq4at2/NRvbax49TwfDw

[3.4922]

[3.5064]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CnCSHdI5+5p+L6+r/YITxQ
replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm at line 15

[3.5538]→[2.707:720](∅→∅)

"jobname",

[3.5538]

[3.5548]

"project",
{ data_type => "text", is_nullable => 0, size => undef },
"jobset",
{ data_type => "text", is_nullable => 0, size => undef },
"attrname",
replacement in src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm at line 51

[3.6194]→[2.1150:1292](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gp6ZZpDA2VzgnNE9NX99dA

[3.6194]

[3.6336]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Odp6qymLlNXbsD7VOQ7PAQ
file addition: Jobsetinputs.pm (----------)

[3.3332]

package HydraFrontend::Schema::Jobsetinputs;

use strict;
use warnings;

use base 'DBIx::Class';

__PACKAGE__->load_components("Core");
__PACKAGE__->table("jobSetInputs");
__PACKAGE__->add_columns(
"project",
{ data_type => "text", is_nullable => 0, size => undef },
"job",
{ data_type => "text", is_nullable => 0, size => undef },
"name",
{ data_type => "text", is_nullable => 0, size => undef },
"type",
{ data_type => "text", is_nullable => 0, size => undef },
"uri",
{ data_type => "text", is_nullable => 0, size => undef },
"revision",
{ data_type => "integer", is_nullable => 0, size => undef },
"tag",
{ data_type => "text", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("project", "job", "name");
__PACKAGE__->has_many(
"jobsets",
"HydraFrontend::Schema::Jobsets",
{
"foreign.name" => "self.job",
"foreign.nixexprinput" => "self.name",
"foreign.project" => "self.project",
},
);
__PACKAGE__->belongs_to(
"jobset",
"HydraFrontend::Schema::Jobsets",
{ name => "job", project => "project" },
);

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pzKFsX3b5wTNZvo8t3WTDg

# You can replace this text with custom content, and it will be preserved on regeneration
1;
file addition: Jobsets.pm (----------)

[3.3332]

package HydraFrontend::Schema::Jobsets;

use strict;
use warnings;

use base 'DBIx::Class';

__PACKAGE__->load_components("Core");
__PACKAGE__->table("jobSets");
__PACKAGE__->add_columns(
"name",
{ data_type => "text", is_nullable => 0, size => undef },
"project",
{ data_type => "text", is_nullable => 0, size => undef },
"description",
{ data_type => "text", is_nullable => 0, size => undef },
"nixexprinput",
{ data_type => "text", is_nullable => 0, size => undef },
"nixexprpath",
{ data_type => "text", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("project", "name");
__PACKAGE__->belongs_to(
"project",
"HydraFrontend::Schema::Projects",
{ name => "project" },
);
__PACKAGE__->belongs_to(
"jobsetinput",
"HydraFrontend::Schema::Jobsetinputs",
{ job => "name", name => "nixexprinput", project => "project" },
);
__PACKAGE__->has_many(
"jobsetinputs",
"HydraFrontend::Schema::Jobsetinputs",
{ "foreign.job" => "self.name", "foreign.project" => "self.project" },
);

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pEIAO9lDM+lMKLCLGWRdXg

# You can replace this text with custom content, and it will be preserved on regeneration
1;
file addition: Projects.pm (----------)

[3.3332]

package HydraFrontend::Schema::Projects;

use strict;
use warnings;

use base 'DBIx::Class';

__PACKAGE__->load_components("Core");
__PACKAGE__->table("projects");
__PACKAGE__->add_columns(
"name",
{ data_type => "text", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("name");
__PACKAGE__->has_many(
"jobsets",
"HydraFrontend::Schema::Jobsets",
{ "foreign.project" => "self.name" },
);

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cpO0BGfChpnpm7KBKkSUjw

# You can replace this text with custom content, and it will be preserved on regeneration
1;
replacement in src/HydraFrontend/lib/HydraFrontend/Schema.pm at line 11

[3.6498]→[2.1293:1435](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fayli8dtSdcAYhfKSZnJwg

[3.6498]

[3.6640]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UAjA2VmMoOSjiHk0NUzLfQ
replacement in src/HydraFrontend/root/index.tt at line 9

[3.919]→[3.919:1003](∅→∅)

<tr><th></th><th>Id</th><th>Job</th><th>Timestamp</th><th>Description</th></tr>

[3.919]

[3.1003]

<tr><th></th><th>#</th><th>Job</th><th>Timestamp</th><th>Description</th></tr>
file move: hydra.schema (----------) → hydra.sql (----------)

[3.4]

[3.1]
replacement in src/hydra.sql at line 4

[3.176]→[2.1829:1862](∅→∅)

jobName text not null,

[3.176]

[3.209]

-- Info about the inputs.
project text not null, -- !!! foreign key
jobSet text not null, -- !!! foreign key
attrName text not null,

-- !!! list all the inputs / arguments

-- Info about the build result.
edit in src/hydra.sql at line 51

[3.1280]

create table projects (
name text primary key not null
);

-- A jobset consists of a set of inputs (e.g. SVN repositories), one
-- of which contains a Nix expression containing an attribute set
-- describing build jobs.
create table jobSets (
name text not null,
project text not null,
description text,
nixExprInput text not null, -- name of the jobSetInput containing the Nix expression
nixExprPath text not null, -- relative path of the Nix expression
primary key (project, name),
foreign key (project) references projects(name) on delete cascade, -- ignored by sqlite
foreign key (project, name, nixExprInput) references jobSetInputs(project, job, name)
);

create table jobSetInputs (
project text not null,
job text not null,
name text not null,
type text not null, -- "svn", "cvs", "path", "file"
uri text,
revision integer, -- for svn
tag text, -- for cvs
primary key (project, job, name),
foreign key (project, job) references jobSets(project, name) on delete cascade -- ignored by sqlite
);
edit in src/scheduler.pl at line 5

[3.1363]→[3.1363:1372](∅→∅)

use DBI;
edit in src/scheduler.pl at line 6

[3.1392]

[3.1392]

use HydraFrontend::Schema;
replacement in src/scheduler.pl at line 9

[3.1394]→[3.1394:1424](∅→∅)

my $jobsFile = "../test.nix";

[3.1394]

[3.1424]

my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
replacement in src/scheduler.pl at line 12

[3.1426]→[3.1426:1492](∅→∅)

my $dbh = DBI->connect("dbi:SQLite:dbname=hydra.sqlite", "", "");

[3.1426]

[3.1492]

sub buildJob {
my ($project, $jobset, $jobName, $drvPath, $outPath) = @_;
replacement in src/scheduler.pl at line 15

[3.1493]→[3.1493:2133](∅→∅),[3.2133]→[2.2171:2304](∅→∅),[2.2304]→[3.2263:2311](∅→∅),[3.2263]→[3.2263:2311](∅→∅)

my $jobsXml = `nix-env -f $jobsFile --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*"`
or die "cannot evaluate the Nix expression containing the job definitions: $?";

print "$jobsXml";

my $jobs = XMLin($jobsXml, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";

foreach my $jobName (keys %{$jobs->{item}}) {
my $job = $jobs->{item}->{$jobName};
my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
print "JOB: $jobName ($description)\n";

my $outPath = $job->{outPath};
my $drvPath = $job->{drvPath};

if (scalar(@{$dbh->selectall_arrayref("select * from builds where jobName = ? and outPath = ?", {}, $jobName, $outPath)}) > 0) {
print " already done\n";
next;

[3.1493]

[3.2311]

if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {
print " already done\n";
return;
replacement in src/scheduler.pl at line 30

[2.2542]→[2.2542:2607](∅→∅)

my $res = system("nix-build $jobsFile --attr $jobName");

[2.2542]

[2.2607]

print " BUILDING\n";

my $res = system("nix-store --realise $drvPath");
edit in src/scheduler.pl at line 39

[3.2422]

[3.2422]

$db->txn_do(sub {
my $build = $db->resultset('Builds')->create(
{ timestamp => time()
, project => $project->name
, jobset => $jobset->name
, attrname => $jobName
, drvpath => $drvPath
, outpath => $outPath
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime
});
});

return 0;

my $dbh, my $description, my $jobName;
edit in src/scheduler.pl at line 88

[3.3635]

[3.3635]

}

sub fetchInput {
my ($input, $inputInfo) = @_;
my $type = $input->type;
my $uri = $input->uri;

if ($type eq "path") {
my $storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;
print " copied to $storePath\n";
$$inputInfo{$input->name} = {storePath => $storePath};
}

else {
die "input `" . $input->type . "' has unknown type `$type'";
}
edit in src/scheduler.pl at line 108

[3.3637]

sub checkJobSet {
my ($project, $jobset) = @_;

my $inputInfo = {};

foreach my $input ($jobset->jobsetinputs) {
print " INPUT ", $input->name, " (", $input->type, " ", $input->uri, ")\n";
fetchInput($input, $inputInfo);
}

die unless defined $inputInfo->{$jobset->nixexprinput};

my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;

print " EVALUATING $nixExprPath\n";

my $jobsXml = `nix-instantiate $nixExprPath --eval-only --strict --xml`
or die "cannot evaluate the Nix expression containing the jobs: $?";

#print "$jobsXml";

my $jobs = XMLin($jobsXml,
ForceArray => [qw(value)],
KeyAttr => ['name'],
SuppressEmpty => '',
ValueAttr => [value => 'value'])
or die "cannot parse XML output";

die unless defined $jobs->{attrs};

foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
print " JOB $jobName\n";

my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};

my $extraArgs = "";

# If the expression is a function, then look at its formal
# arguments and see if we can supply them.
if (defined $jobExpr->{function}->{attrspat}) {
foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
print " needs input $argName\n";
die "missing input `$argName'" if !defined $inputInfo->{$argName};
$extraArgs .= " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'";
}
}

# Instantiate the store derivation.
my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs`
or die "cannot evaluate the Nix expression containing the job definitions: $?";
chomp $drvPath;

# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs`
or die "cannot get information about the job: $?";

my $info = XMLin($infoXml, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";

my $job = $info->{item};
die unless !defined $job || $job->{system} ne $jobName;

my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
die unless $job->{drvPath} eq $drvPath;
my $outPath = $job->{outPath};

buildJob($project, $jobset, $jobName, $drvPath, $outPath);
}
}

sub checkJobs {

foreach my $project ($db->resultset('Projects')->all) {
print "PROJECT ", $project->name, "\n";
foreach my $jobset ($project->jobsets->all) {
print " JOBSET ", $jobset->name, "\n";
checkJobSet($project, $jobset);
}
}

}

checkJobs;
file addition: test.sql (----------)

[3.4]

insert into projects(name) values('patchelf');
insert into jobSets(project, name, description, nixExprInput, nixExprPath) values('patchelf', 'trunk', 'PatchELF', 'patchelfSrc', 'release.nix');
insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'patchelfSrc', 'path', '/home/eelco/Dev/patchelf-wc');
insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'nixpkgs', 'path', '/home/eelco/Dev/nixpkgs-wc');
insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'release', 'path', '/home/eelco/Dev/release');

--insert into projects(name) values('nixpkgs');
--insert into jobSets(project, name) values('nixpkgs', 'trunk');
--insert into jobSets(project, name) values('nixpkgs', 'stdenv-branch');