hydra: add some admin for adding/enabling/etc build machines

Dependencies

• [2] 7Z7HOZQ3 * Remove the triggers to simulate foreign key constraints on SQLite,
• [3] SGNXIOI4 Hydra/32: Add option to force evaluation of a certain jobset via web interface (for admins only)
• [4] YJAHR4FU * jQuery 1.8.4.
• [5] EFWN7JBV * Added a status page that shows all the currently executing build steps.
• [6] F2YSY4BK update jquery versions in templates
• [7] QL55ECJ6 - adapted ui for hydra, more in line with nixos.org website
• [8] BA46C5LN * Pretty-print the logs.
• [9] QB3LWT7N * Ouch.
• [10] UVMFS73T * Some jQuery / CSS hackery.
• [11] ZWCTAZGL added newsitems, added some admin options to clear various caches.
• [12] ODNCGFQ5 * Improved the navigation bar: don't include all projects (since that
• [13] QT4FO2HP refactored admin controller, using chains, to avoid using requireadmin on each endpoint
• [14] FPK5LF53 * Put the project-related actions in a separate controller. Put the
• [15] 6FRLEP4P first try for timeline of last 24 hours in hydra
• [*] J5UVLXOK * Start of a basic Catalyst web interface.
• [*] D5QIOJGP * Move everything up one directory.
• [*] IK53RV4V
• [*] KAZWI5G4 * hydra: buildpage, show changes since last build/successful build
• [*] N22GPKYT * Put info about logs / build products in the DB.
• [*] 2M7J26V4 inital version of links to diff in scm
• [*] RBNQKATL * Adding persistant releases. A release is a named set of builds.

Change contents

• edit in src/lib/Hydra/Controller/Admin.pm at line 9
  [3.30]

  [6.0]
  + use Data::Dump qw(dump);

  +

  + sub nixMachines {

  + my ($c) = @_;

  + my $result = '';

  +

  + foreach my $machine ($c->model("DB::BuildMachines")->all) {

  + if($machine->enabled) {

  + $result = $result . $machine->username . '@'. $machine->hostname . ' ';

  + foreach my $system ($machine->buildmachinesystemtypes) {

  + $result = $result . $system->system .',';

  + }

  + chop $result;

  + $result = $result . ' '. $machine->ssh_key . ' ' . $machine->maxconcurrent . ' '. $machine->speedfactor . ' ' . $machine->options . "\n";

  + }

  + }

  + return $result;

  + }

  +

  + sub saveNixMachines {

  + my ($c) = @_;

  +

  + die("File not writable: /etc/nix.machines") if ! -w "/etc/nix.machines" ;
• edit in src/lib/Hydra/Controller/Admin.pm at line 33
  [6.1]

  [6.1]
  + open (NIXMACHINES, '>/etc/nix.machines') or die("Could not write to /etc/nix.machines");

  + print NIXMACHINES nixMachines($c);

  + close (NIXMACHINES);

  + }

  +
• edit in src/lib/Hydra/Controller/Admin.pm at line 41
  [6.239]

  [6.62]
  + $c->stash->{admin} = 1;
• edit in src/lib/Hydra/Controller/Admin.pm at line 46
  [6.142]

  [6.240]
  + $c->stash->{machines} = [$c->model('DB::BuildMachines')->search(

  + {},

  + { order_by => "hostname"

  + , '+select' => ["(select bs.stoptime from buildsteps as bs where bs.machine = (me.username || '\@' || me.hostname) and not bs.stoptime is null order by bs.stoptime desc limit 1)"]

  + , '+as' => ['idle']

  + })];

  + $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(

  + { 'me.busy' => 1, 'schedulingInfo.busy' => 1 },

  + { join => [ 'schedulingInfo', 'build' ]

  + , order_by => [ 'machine', 'outpath' ]

  + } ) ];
• edit in src/lib/Hydra/Controller/Admin.pm at line 58
  [6.280]

  [6.280]
  + }

  +

  + sub machines : Chained('admin') PathPart('machines') Args(0) {

  + my ($self, $c) = @_;

  + $c->stash->{machines} = [$c->model('DB::BuildMachines')->search({}, {order_by => "hostname"})];

  + $c->stash->{systems} = [$c->model('DB::SystemTypes')->search({}, {select => ["system"], order_by => "system" })];

  + $c->stash->{nixMachines} = nixMachines($c);

  + $c->stash->{nixMachinesWritable} = (-e "/etc/nix.machines" && -w "/etc/nix.machines");

  +

  + $c->stash->{template} = 'machines.tt';

  + }

  +

  + sub machine : Chained('admin') PathPart('machine') CaptureArgs(1) {

  + my ($self, $c, $machineName) = @_;

  +

  + requireAdmin($c);

  +

  + my $machine = $c->model('DB::BuildMachines')->find($machineName)

  + or notFound($c, "Machine $machineName doesn't exist.");

  +

  + $c->stash->{machine} = $machine;
• edit in src/lib/Hydra/Controller/Admin.pm at line 81
  [6.283]

  [6.143]
  + sub machine_edit : Chained('machine') PathPart('edit') Args(0) {

  + my ($self, $c) = @_;

  + $c->stash->{template} = 'machine.tt';

  + $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})];

  + $c->stash->{edit} = 1;

  + }

  +

  + sub machine_edit_submit : Chained('machine') PathPart('submit') Args(0) {

  + my ($self, $c) = @_;

  + requirePost($c);

  +

  + txn_do($c->model('DB')->schema, sub {

  + updateMachine($c, $c->stash->{machine}) ;

  + });

  + saveNixMachines($c);

  + $c->res->redirect("/admin/machines");

  + }

  +

  + sub updateMachine {

  + my ($c, $machine) = @_;

  +

  + my $hostname = trim $c->request->params->{"hostname"};

  + my $username = trim $c->request->params->{"username"};

  + my $maxconcurrent = trim $c->request->params->{"maxconcurrent"};

  + my $speedfactor = trim $c->request->params->{"speedfactor"};

  + my $ssh_key = trim $c->request->params->{"ssh_key"};

  + my $systems = $c->request->params->{"systems"} ;

  +

  + error($c, "Invalid or empty username.") if $username eq "";

  + error($c, "Max concurrent builds should be an integer > 0.") if $maxconcurrent eq "" || ! $maxconcurrent =~ m/[0-9]+/;

  + error($c, "Speed factor should be an integer > 0.") if $speedfactor eq "" || ! $speedfactor =~ m/[0-9]+/;

  + error($c, "Invalid or empty SSH key.") if $ssh_key eq "";

  +

  + $machine->update(

  + { username => $username

  + , maxconcurrent => $maxconcurrent

  + , speedfactor => $speedfactor

  + , ssh_key => $ssh_key

  + });

  + $machine->buildmachinesystemtypes->delete_all;

  + if(ref($systems) eq 'ARRAY') {

  + for my $s (@$systems) {

  + $machine->buildmachinesystemtypes->create({ system => $s}) ;

  + }

  + } else {

  + $machine->buildmachinesystemtypes->create({ system => $systems}) ;

  + }

  + }

  +

  + sub create_machine : Chained('admin') PathPart('create-machine') Args(0) {

  + my ($self, $c) = @_;

  +

  + requireAdmin($c);

  +

  + $c->stash->{template} = 'machine.tt';

  + $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})];

  + $c->stash->{edit} = 1;

  + $c->stash->{create} = 1;

  + }

  +

  +

  + sub create_machine_submit : Chained('admin') PathPart('create-machine/submit') Args(0) {

  + my ($self, $c) = @_;

  +

  + requireAdmin($c);

  +

  + my $hostname = trim $c->request->params->{"hostname"};

  + error($c, "Invalid or empty hostname.") if $hostname eq "";

  +

  + txn_do($c->model('DB')->schema, sub {

  + my $machine = $c->model('DB::BuildMachines')->create(

  + { hostname => $hostname });

  + updateMachine($c, $machine);

  + });

  + $c->res->redirect("/admin/machines");

  + }

  +

  + sub machine_delete : Chained('machine') PathPart('delete') Args(0) {

  + my ($self, $c) = @_;

  + requirePost($c);

  +

  + txn_do($c->model('DB')->schema, sub {

  + $c->stash->{machine}->delete;

  + });

  + saveNixMachines($c);

  + $c->res->redirect("/admin/machines");

  + }

  +

  + sub machine_enable : Chained('machine') PathPart('enable') Args(0) {

  + my ($self, $c) = @_;

  + $c->stash->{machine}->update({ enabled => 1});

  + saveNixMachines($c);

  + $c->res->redirect("/admin/machines");

  + }

  +

  + sub machine_disable : Chained('machine') PathPart('disable') Args(0) {

  + my ($self, $c) = @_;

  + $c->stash->{machine}->update({ enabled => 0});

  + saveNixMachines($c);

  + $c->res->redirect("/admin/machines");

  + }

  +
• replacement in src/lib/Hydra/Controller/Admin.pm at line 197
  [6.690]→[6.690:723](∅→∅)
  − $c->res->redirect("/admin");
  [6.690]

  [6.723]
  + $c->res->redirect("/admin")
• replacement in src/lib/Hydra/Controller/Root.pm at line 101
  [5.544]→[5.544:584](∅→∅)
  − { join => [ 'schedulingInfo' ]
  [5.544]

  [5.584]
  + { join => [ 'schedulingInfo', 'build' ]
• file addition: BuildMachineSystemTypes.pm (----------)
  [18.477]
  + package Hydra::Schema::BuildMachineSystemTypes;

  +

  + # Created by DBIx::Class::Schema::Loader

  + # DO NOT MODIFY THE FIRST PART OF THIS FILE

  +

  + use strict;

  + use warnings;

  +

  + use base 'DBIx::Class::Core';

  +

  +

  + =head1 NAME

  +

  + Hydra::Schema::BuildMachineSystemTypes

  +

  + =cut

  +

  + __PACKAGE__->table("BuildMachineSystemTypes");

  +

  + =head1 ACCESSORS

  +

  + =head2 hostname

  +

  + data_type: text

  + default_value: undef

  + is_foreign_key: 1

  + is_nullable: 0

  + size: undef

  +

  + =head2 system

  +

  + data_type: text

  + default_value: undef

  + is_nullable: 0

  + size: undef

  +

  + =cut

  +

  + __PACKAGE__->add_columns(

  + "hostname",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_foreign_key => 1,

  + is_nullable => 0,

  + size => undef,

  + },

  + "system",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_nullable => 0,

  + size => undef,

  + },

  + );

  + __PACKAGE__->set_primary_key("hostname", "system");

  +

  + =head1 RELATIONS

  +

  + =head2 hostname

  +

  + Type: belongs_to

  +

  + Related object: L<Hydra::Schema::BuildMachines>

  +

  + =cut

  +

  + __PACKAGE__->belongs_to(

  + "hostname",

  + "Hydra::Schema::BuildMachines",

  + { hostname => "hostname" },

  + {},

  + );

  +

  +

  + # Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-08 13:47:26

  + # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/voQZLNESTotUOWRbg4WA

  +

  +

  + # You can replace this text with custom content, and it will be preserved on regeneration

  + 1;
• file addition: BuildMachines.pm (----------)
  [18.477]
  + package Hydra::Schema::BuildMachines;

  +

  + # Created by DBIx::Class::Schema::Loader

  + # DO NOT MODIFY THE FIRST PART OF THIS FILE

  +

  + use strict;

  + use warnings;

  +

  + use base 'DBIx::Class::Core';

  +

  +

  + =head1 NAME

  +

  + Hydra::Schema::BuildMachines

  +

  + =cut

  +

  + __PACKAGE__->table("BuildMachines");

  +

  + =head1 ACCESSORS

  +

  + =head2 hostname

  +

  + data_type: text

  + default_value: undef

  + is_nullable: 0

  + size: undef

  +

  + =head2 username

  +

  + data_type: text

  + default_value: undef

  + is_nullable: 0

  + size: undef

  +

  + =head2 ssh_key

  +

  + data_type: text

  + default_value: undef

  + is_nullable: 0

  + size: undef

  +

  + =head2 options

  +

  + data_type: text

  + default_value: undef

  + is_nullable: 0

  + size: undef

  +

  + =head2 maxconcurrent

  +

  + data_type: integer

  + default_value: 2

  + is_nullable: 0

  + size: undef

  +

  + =head2 speedfactor

  +

  + data_type: integer

  + default_value: 1

  + is_nullable: 0

  + size: undef

  +

  + =head2 enabled

  +

  + data_type: integer

  + default_value: 1

  + is_nullable: 0

  + size: undef

  +

  + =cut

  +

  + __PACKAGE__->add_columns(

  + "hostname",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_nullable => 0,

  + size => undef,

  + },

  + "username",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_nullable => 0,

  + size => undef,

  + },

  + "ssh_key",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_nullable => 0,

  + size => undef,

  + },

  + "options",

  + {

  + data_type => "text",

  + default_value => undef,

  + is_nullable => 0,

  + size => undef,

  + },

  + "maxconcurrent",

  + { data_type => "integer", default_value => 2, is_nullable => 0, size => undef },

  + "speedfactor",

  + { data_type => "integer", default_value => 1, is_nullable => 0, size => undef },

  + "enabled",

  + { data_type => "integer", default_value => 1, is_nullable => 0, size => undef },

  + );

  + __PACKAGE__->set_primary_key("hostname");

  +

  + =head1 RELATIONS

  +

  + =head2 buildmachinesystemtypes

  +

  + Type: has_many

  +

  + Related object: L<Hydra::Schema::BuildMachineSystemTypes>

  +

  + =cut

  +

  + __PACKAGE__->has_many(

  + "buildmachinesystemtypes",

  + "Hydra::Schema::BuildMachineSystemTypes",

  + { "foreign.hostname" => "self.hostname" },

  + );

  +

  +

  + # Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-11 12:58:16

  + # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gje+VA73Hghl7JXp+Fl8pw

  +

  +

  + # You can replace this text with custom content, and it will be preserved on regeneration

  + 1;
• edit in src/root/admin.tt at line 15
  [6.9263]→[6.9263:9385](∅→∅)
  − <li>[% INCLUDE maybeLink uri = c.uri_for(c.controller('Admin').action_for('managenews')) content = "News" %]</li>
• edit in src/root/admin.tt at line 17
  [6.9392]

  [6.9392]
  + <h2>Status</h2>
• edit in src/root/admin.tt at line 19
  [6.9393]

  [6.9393]
  + [% FOREACH m IN machines %]

  + <table style="width: 40em;">

  + <thead>

  + <tr>

  + <th colspan="5">

  + [% IF m.enabled == 1 %]

  + [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'disable' ) content='-' %]

  + [% ELSE %]

  + [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'enable' ) content='+' %]

  + [% END %]

  + [% m.hostname %] <tt>[% FOREACH ms IN m.buildmachinesystemtypes %] [% ms.system %][% END %]</tt>

  + </th>

  + </tr>

  + </thead>

  + <tbody>

  + [% idle = 1 %]

  + [% FOREACH step IN steps %]

  + [% IF step.machine.match('@(.*)').0 == m.hostname %]

  + [% idle = 0 %]

  + <tr>

  + <td><tt>[% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %]</tt></td>

  + <td><tt>[% step.system %]</tt></td>

  + <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>

  + <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>

  + <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>

  + </tr>

  + [% END %]

  + [% END %]

  + [% IF idle == 1 %]

  + <tr><td colspan="5">Idle since [% INCLUDE renderDuration duration = curTime - m.get_column('idle') %]</td></tr>

  + [% END %]

  + </tbody>

  + </table>

  +

  + [% END %]

  +

  +
• edit in src/root/common.tt at line 337
  [20.3391]
  + [% BLOCK hydraStatus %]

  + <table class="tablesorter">

  + <thead>

  + <tr><th>Machine</th><th>Job</th><th>Type</th><th>Build</th><th>Step</th><th>What</th><th>Since</th></tr>

  + </thead>

  + <tbody>

  + [% FOREACH step IN steps %]

  + <tr>

  + <td><tt>[% IF step.machine; step.machine.match('@(.*)').0; ELSE; 'localhost'; END %]</tt></td>

  + <td><tt>[% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %]</tt></td>

  + <td><tt>[% step.system %]</tt></td>

  + <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>

  + <td><a href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr) %]">[% step.stepnr %]</a></td>

  + <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>

  + <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>

  + </tr>

  + [% END %]

  + </tbody>

  + </table>

  + [% END %]
• replacement in src/root/layout.tt at line 19
  [6.207]→[4.259:362](∅→∅)
  − <script type="text/javascript" src="/static/js/jquery/js/jquery-ui-1.8.4.custom.min.js"></script>
  [6.207]

  [6.1805]
  + <script type="text/javascript" src="/static/js/jquery/js/jquery-ui-1.8.5.custom.min.js"></script>
• replacement in src/root/navbar.tt at line 27
  [6.15749]→[6.2824:2841](∅→∅)
  − [% IF project %]
  [6.15749]

  [6.16330]
  + [% IF project || admin %]
• edit in src/root/navbar.tt at line 76
  [6.18024]

  [6.18024]
  +

  + [% IF admin %]

  + [% WRAPPER makeSubMenu title="Admin" %]

  + [% INCLUDE makeLink

  + uri = c.uri_for(c.controller('Admin').action_for('machines'))

  + title = "Machines" %]

  + [% INCLUDE makeLink

  + uri = c.uri_for(c.controller('Admin').action_for('managenews'))

  + title = "News" %]

  + [% END %]

  + [% END %]
• replacement in src/root/status.tt at line 6
  [5.1245]→[5.1245:1997](∅→∅)
  − <table class="tablesorter">

  − <thead>

  − <tr><th>Machine</th><th>Type</th><th>Build</th><th>Step</th><th>What</th><th>Since</th></tr>

  − </thead>

  − <tbody>

  − [% FOREACH step IN steps %]

  − <tr>

  − <td><tt>[% IF step.machine; step.machine.match('@(.*)').0; ELSE; 'localhost'; END %]</tt></td>

  − <td><tt>[% step.system %]</tt></td>

  − <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>

  − <td><a href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr) %]">[% step.stepnr %]</a></td>

  − <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>

  − <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>

  − </tr>

  − [% END %]

  − </tbody>

  − </table>
  [5.1245]

  [5.1997]
  + [% INCLUDE hydraStatus %]
• edit in src/sql/hydra.sql at line 486
  [6.11975]

  [22.864]
  + );

  +

  + create table BuildMachines (

  + hostname text primary key NOT NULL,

  + username text NOT NULL,

  + ssh_key text NOT NULL,

  + options text NOT NULL,

  + maxconcurrent integer DEFAULT 2 NOT NULL,

  + speedfactor integer DEFAULT 1 NOT NULL,

  + enabled integer DEFAULT 1 NOT NULL
• edit in src/sql/hydra.sql at line 498
  [23.5025]

  [2.2]
  + create table BuildMachineSystemTypes (

  + hostname text NOT NULL,

  + system text NOT NULL,

  + primary key (hostname, system),

  + foreign key (hostname) references BuildMachines(hostname) on delete cascade

  + );