hydra-queue-runner: Improved scheduling

[?]

Sep 21, 2013, 2:47 PM

GEADFVZ5LXXFIE3VIP4UJ4AEI2VX57DXER47JA4IHH5BG3QNPAEAC

Dependencies

[2] 2PWOXJTX
[3] TFLAR4KA hydra-queue-runner: Start as many builds as possible on each iteration
[4] TPSCSZKX Speed up findBuildDependencyInQueue
[5] I7UELKBV hydra-queue-runner: Don't unlock builds we just started
[6] 3BTJRSU3 GitInput.pm: Don't do a chdir to the Git clone
[7] D7X6XTKQ Integrate the "Job status" and "All jobs" tabs
[8] Q5HZWFCY Add support for darcs repositories.
[9] AMI4DGBK Don't trigger evaluation of disabled jobsets
[10] 6ZB4CIW6 Security: Improve checking of build products
[11] 7VWDMKAZ hydra-queue-runner: don't clutter the system log with debug message
[12] ECBA3GQO * Make the schema class names match the case of the SQL table names.
[13] 7YBYT2LQ
[14] RNNLIVYY Respect SystemTypes if defined
[15] M4TBFHHJ "limit" -> "rows"
[16] MREXL5RT Whoops
[17] NREF6YOA * Don't start more builds concurrently than allowed for each system
[18] DQD7JMSU * Fix the terminology.
[19] BDSD2JLV * Speed up manifest generation.
[20] YTIDBFGU Drop unused "disabled" columns
[21] D7TT2BNK
[22] SB2V735V Keep track of the database schema version
[23] MOX7XJ2E Merge the BuildSchedulingInfo table into the Builds table
[24] 6QRHXIM3 * Speed up the jobset index page. Especially the query to get the
[25] PYTQMQKD Hydra/17: in queue runner, prefer builds in the queue that are a dependency of another build (with higher priority)
[26] AK2UZDS2 Jobset page: Add a new tab to show job status in a matrix
[27] YEXD7CBK Fix findBuildDependencyInQueue
[28] Y4B2MURV
[29] X27GNHDV * Basic job info in the database.
[30] FGQPXZIX hydra: make nr of build to keep configurable per jobset
[31] D6YQQQCN * Don't ignore SIGCHLD after all, Perl doesn't like it. Just do
[32] R6B5CAFF Let Builds.timestamp refer to the time the build was added
[33] JAH3UPWA Support revision control systems via plugins
[34] LZO3C2KI * Hack around those SQLite timeouts: just retry the transaction.
[35] DTXTS7LN * Speed up findBuildDependencyInQueue by doing only one SQL query for
[36] OV7F5M3E Merge branch 'queue-17'
[37] QLOLZHRX Allow a per-jobset check interval
[38] AFTXA575 * $HYDRA_DATA environment variable.
[39] QTFVCDIF added hide feature for project/jobset
[40] PCKLFRT5 Support push notification of repository changes
[41] LZVO64YG Merge in the first bits of the API work
[42] L2E6EVE2 * Merged the Build and Job tables.
[43] Y6AHH4TH Remove the logfile and logSize columns from the database
[44] 3PNG7NIB Remove trailing whitespace
[45] SA5ZZ3I4 hydra-queue-runner: Use nix.machines instead of the SystemTypes table to determine how many build jobs are allowed per system type.
[46] QZLMDKMU * Queue runner: don't start scheduled builds builds if they belong to
[47] 3E6IP3R3 * Add the name of the jobset to ReleaseSetJobs, otherwise we can't
[48] JM3DPYOM generated schema with new dbix class schema loader, grrrrrr
[49] RFE6T5LG * Store jobset evaluations in the database explicitly. This includes
[50] QTC3SYBM Jobset page: Load the jobs and status tabs on demand
[51] TWVSALRL * Allow the maximum number of concurrent builds per platform to be
[*] 3HZY24CX * Make jobsets viewable under
[*] ODNCGFQ5 * Improved the navigation bar: don't include all projects (since that
[*] FPK5LF53 * Put the project-related actions in a separate controller. Put the
[*] 2GK5DOU7 * Downloading closures.
[*] OX6NYJDV Split viewing and editing a jobset
[*] N22GPKYT * Put info about logs / build products in the DB.
[*] KN3VYE5P * Cleaned up the foreign key constraints.
[*] D5QIOJGP * Move everything up one directory.

Change contents

replacement in src/lib/Hydra/Controller/Jobset.pm at line 53

[10.174]→[10.0:118](∅→∅)

($c->stash->{latestEval}) = $c->stash->{jobset}->jobsetevals->search({}, { rows => 1, order_by => ["id desc"] });

[10.174]

[10.7208]

$c->stash->{latestEval} = $c->stash->{jobset}->jobsetevals->search({}, { rows => 1, order_by => ["id desc"] })->single;

$c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
replacement in src/lib/Hydra/Controller/Jobset.pm at line 166

[10.826]→[10.826:1096](∅→∅)

foreach my $b (@builds) {
my $jobName = $b->get_column('job');
$evals->{$eval->id}->{$jobName} =
{ id => $b->id, finished => $b->finished, buildstatus => $b->buildstatus };
$jobs{$jobName} = 1;
$nrBuilds++;
}
last if $nrBuilds >= 10000;

[10.826]

[10.1858]

foreach my $b (@builds) {
my $jobName = $b->get_column('job');
$evals->{$eval->id}->{$jobName} =
{ id => $b->id, finished => $b->finished, buildstatus => $b->buildstatus };
$jobs{$jobName} = 1;
$nrBuilds++;
}
last if $nrBuilds >= 10000;
replacement in src/lib/Hydra/Controller/Jobset.pm at line 177

[7.144]→[7.144:378](∅→∅)

$c->stash->{showInactive} = 1;
foreach my $job ($c->stash->{jobset}->jobs->search({ name => { ilike => $filter } })) {
next if defined $jobs{$job->name};
$c->stash->{inactiveJobs}->{$job->name} = $jobs{$job->name} = 1;
}

[7.144]

[7.378]

$c->stash->{showInactive} = 1;
foreach my $job ($c->stash->{jobset}->jobs->search({ name => { ilike => $filter } })) {
next if defined $jobs{$job->name};
$c->stash->{inactiveJobs}->{$job->name} = $jobs{$job->name} = 1;
}
edit in src/lib/Hydra/Controller/Jobset.pm at line 214

[54.705]

[54.705]

$c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
edit in src/lib/Hydra/Controller/Jobset.pm at line 293

[9.174]

[54.2272]

, schedulingshares => int($c->stash->{params}->{schedulingshares})
edit in src/lib/Hydra/Controller/Project.pm at line 204

[54.5027]

[54.5027]

$c->stash->{totalShares} = getTotalShares($c->model('DB')->schema);
replacement in src/lib/Hydra/Helper/Nix.pm at line 23

[10.1545]→[6.0:35](∅→∅)

captureStdoutStderr run grab);

[10.1545]

[10.77]

captureStdoutStderr run grab
getTotalShares);
edit in src/lib/Hydra/Helper/Nix.pm at line 537

[6.1010]

[10.850]

sub getTotalShares {
my ($db) = @_;
return $db->resultset('Jobsets')->search(
{ 'project.enabled' => 1, 'me.enabled' => 1 },
{ join => 'project', select => { sum => 'schedulingshares' }, as => 'sum' })->single->get_column('sum');
}
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 17

[8.3893]

[8.3893]

=head1 COMPONENTS LOADED
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 20

[8.3894]

[8.3894]

=over 4

=item * L<Hydra::Component::ToJSON>

=back

=cut

__PACKAGE__->load_components("+Hydra::Component::ToJSON");
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 39

[8.4007]→[8.4007:4060](∅→∅)

data_type: 'text'
is_nullable: 0

=head2 branch
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 57

[8.4266]

[8.4266]

=head2 revcount
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 60

[8.4267]

[8.4267]

data_type: 'integer'
is_nullable: 0
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 70

[8.4412]→[8.4412:4474](∅→∅)

"revcount",
{ data_type => "integer", is_nullable => 0 },
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 74

[8.4595]

[8.4595]

"revcount",
{ data_type => "integer", is_nullable => 0 },
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 84

[8.4645]→[8.4645:4665](∅→∅)

=item * L</branch>
replacement in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 93

[8.4751]→[8.4751:4893](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fx3yosWMmJ+MnvL/dSWtFA

[8.4751]

[8.4893]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-09-20 11:08:50
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Yl1slt3SAizijgu0KUTn0A
edit in src/lib/Hydra/Schema/CachedDarcsInputs.pm at line 97

[8.4894]

[8.4894]

# You can replace this text with custom code or comments, and it will be preserved on regeneration
edit in src/lib/Hydra/Schema/Jobsets.pm at line 119

[10.226]

[10.256]

is_nullable: 0

=head2 schedulingshares

data_type: 'integer'
default_value: 100
edit in src/lib/Hydra/Schema/Jobsets.pm at line 160

[10.316]

[10.2524]

"schedulingshares",
{ data_type => "integer", default_value => 100, is_nullable => 0 },
replacement in src/lib/Hydra/Schema/Jobsets.pm at line 283

[10.3136]→[10.31884:32026](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tsGR8MhZRIUeNwpcVczMUw

[10.3136]

[10.22627]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-09-20 12:15:23
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pD6tGW0Ob3fuA1p0uQnBWw
edit in src/root/edit-jobset.tt at line 3

[57.520]

[57.520]

[% USE format %]
edit in src/root/edit-jobset.tt at line 99

[10.860]

[10.860]

<label class="control-label">Scheduling shares</label>
edit in src/root/edit-jobset.tt at line 101

[10.889]

[57.3705]

<div class="input-append">
<input type="number" class="span3" name="schedulingshares" [% HTML.attributes(value => jobset.schedulingshares) %]/>
</div>
[% IF totalShares %]
<span class="help-inline">([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)</span>
[% END %]
</div>
</div>

<div class="control-group">
<div class="controls">
edit in src/root/jobset.tt at line 3

[54.8624]

[54.8841]

[% USE format %]
edit in src/root/jobset.tt at line 126

[57.9334]

[57.9334]

<th>Scheduling shares:</th>
<td>[% jobset.schedulingshares %] [% IF totalShares %] ([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)[% END %]</td>
</tr>
<tr>
replacement in src/script/hydra-queue-runner at line 31

[10.116]→[5.16:97](∅→∅)

if (!defined $lastTime || $build->starttime < $lastTime - 300) {

[10.116]

[10.558]

if (!defined $lastTime || $build->starttime < $lastTime - 600) {
replacement in src/script/hydra-queue-runner at line 73

[10.113]→[10.113:186](∅→∅)

foreach my $system (${$machines}{$machineName}{'systemTypes'}) {

[10.113]

[10.186]

foreach my $system (@{${$machines}{$machineName}{'systemTypes'}}) {
replacement in src/script/hydra-queue-runner at line 80

[10.23]→[4.209:266](∅→∅)

# Cache scheduled by derivation path to speed up

[10.23]

[4.266]

# Cache scheduled builds by derivation path to speed up
replacement in src/script/hydra-queue-runner at line 84

[4.379]→[4.379:491](∅→∅)

foreach $db->resultset('Builds')->search({ finished => 0, enabled => 1 }, { join => ['project'] });

[4.379]

[4.491]

foreach $db->resultset('Builds')->search({ finished => 0 }, { join => ['project'] });
replacement in src/script/hydra-queue-runner at line 88

[10.139]→[10.350:406](∅→∅)

{ finished => 0, busy => 0, enabled => 1 },

[10.139]

[10.7034]

{ finished => 0, busy => 0 },
edit in src/script/hydra-queue-runner at line 91

[10.37575]

[10.236]

# Get the total number of scheduling shares.
my $totalShares = getTotalShares($db);
replacement in src/script/hydra-queue-runner at line 95

[10.303]→[10.303:429](∅→∅)

# concurrent build for that system type. Choose the highest
# priority builds first, then the oldest builds.

[10.303]

[10.429]

# concurrent build for that system type.
replacement in src/script/hydra-queue-runner at line 106

[10.288]→[10.892:944](∅→∅),[10.388]→[10.892:944](∅→∅),[10.2781]→[10.892:944](∅→∅),[10.892]→[10.892:944](∅→∅)

$extraAllowed = 0 if $extraAllowed < 0;

[10.288]

[10.944]

next if $extraAllowed <= 0;
replacement in src/script/hydra-queue-runner at line 108

[10.945]→[10.945:1004](∅→∅),[10.1004]→[10.540:625](∅→∅),[10.625]→[10.407:494](∅→∅),[10.494]→[3.0:79](∅→∅)

# Select the highest-priority builds to start.
my @builds = $extraAllowed == 0 ? () : $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system, enabled => 1 },
{ join => ['project'], order_by => ["priority DESC", "id"] });

[10.945]

[10.1292]

print STDERR "starting at most $extraAllowed builds for system ${\$system->system}\n";

j: while ($extraAllowed-- > 0) {
replacement in src/script/hydra-queue-runner at line 112

[10.1293]→[3.80:109](∅→∅),[3.109]→[10.686:728](∅→∅),[10.8821]→[10.686:728](∅→∅),[10.728]→[10.428:758](∅→∅),[10.758]→[4.493:578](∅→∅),[4.578]→[10.829:864](∅→∅),[10.829]→[10.829:864](∅→∅)

my $started = 0;
foreach my $build (@builds) {
# Find a dependency of $build that has no queued
# dependencies itself. This isn't strictly necessary,
# but it ensures that Nix builds are done as part of
# their corresponding Hydra builds, rather than as a
# dependency of some other Hydra build.
while (my $dep = findBuildDependencyInQueue($buildsByDrv, $build)) {
$build = $dep;

[10.1293]

[10.864]

my @runnableJobsets = $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system },
{ select => ['project', 'jobset'], distinct => 1 });

next if @runnableJobsets == 0;

my $windowSize = 24 * 3600;
my $totalWindowSize = $windowSize * $max;

my @res;

foreach my $b (@runnableJobsets) {
my $jobset = $db->resultset('Jobsets')->find($b->get_column('project'), $b->get_column('jobset')) or die;

my $duration = $jobset->builds->search(
{ },
{ where => \ ("(finished = 0 or (me.stoptime >= " . (time() - $windowSize) . "))")
, join => 'buildsteps'
, select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
, as => "sum" })->single->get_column("sum") // 0;

# Add a 30s penalty for each started build. This
# is to account for jobsets that have running
# builds but no build steps yet.
$duration += $jobset->builds->search({ finished => 0, busy => 1 })->count * 30;

my $share = $jobset->schedulingshares;
my $delta = ($share / $totalShares) - ($duration / $totalWindowSize);

#printf STDERR "%s:%s: %d s, %.3f%%, allowance = %.3f%%\n", $jobset->get_column('project'), $jobset->name, $duration, $duration / $totalWindowSize, $delta;

push @res, { jobset => $jobset, delta => $delta };
replacement in src/script/hydra-queue-runner at line 145

[10.882]→[10.882:920](∅→∅)

next if $build->busy;

[10.882]

[10.644]

foreach my $r (sort { $b->{delta} <=> $a->{delta} } @res) {
my $jobset = $r->{jobset};
#print STDERR "selected ", $jobset->get_column('project'), ':', $jobset->name, "\n";
replacement in src/script/hydra-queue-runner at line 150

[10.645]→[10.728:790](∅→∅),[10.728]→[10.728:790](∅→∅),[10.790]→[10.1310:1351](∅→∅),[10.790]→[10.60:94](∅→∅),[10.1351]→[10.60:94](∅→∅),[10.60]→[10.60:94](∅→∅),[10.94]→[10.10908:11115](∅→∅),[10.7477]→[10.1104:1149](∅→∅),[10.11115]→[10.1104:1149](∅→∅),[10.1104]→[10.1104:1149](∅→∅)

my $logfile = getcwd . "/logs/" . $build->id;
mkdir(dirname $logfile);
unlink($logfile);
$build->update(
{ busy => 1
, locker => $$
, logfile => $logfile
, starttime => time()
});
push @buildsStarted, $build;

[10.645]

[3.110]

# Select the highest-priority build for this jobset.
my @builds = $jobset->builds->search(
{ finished => 0, busy => 0, system => $system->system },
{ order_by => ["priority DESC", "id"] });

foreach my $build (@builds) {
# Find a dependency of $build that has no queued
# dependencies itself. This isn't strictly necessary,
# but it ensures that Nix builds are done as part of
# their corresponding Hydra builds, rather than as a
# dependency of some other Hydra build.
while (my $dep = findBuildDependencyInQueue($buildsByDrv, $build)) {
$build = $dep;
}
next if $build->busy;

printf STDERR "starting build %d (%s:%s:%s) on %s (jobset allowance = %.3f%%)\n",
$build->id, $build->project->name, $build->jobset->name, $build->job->name, $build->system, $r->{delta};
replacement in src/script/hydra-queue-runner at line 169

[3.111]→[3.111:178](∅→∅)

last if ++$started >= $extraAllowed;
}

[3.111]

[3.178]

my $logfile = getcwd . "/logs/" . $build->id;
mkdir(dirname $logfile);
unlink($logfile);
$build->update(
{ busy => 1
, locker => $$
, logfile => $logfile
, starttime => time()
});
push @buildsStarted, $build;
next j;
}
}
replacement in src/script/hydra-queue-runner at line 183

[3.179]→[3.179:358](∅→∅)

if ($started > 0) {
print STDERR "system type `", $system->system,
"': $nrActive active, $max allowed, started $started builds\n";

[3.179]

[10.1879]

last; # nothing found, give up on this system type
edit in src/script/hydra-queue-runner at line 192

[10.1220]→[2.0:143](∅→∅)

print "starting build $id (", $build->project->name, ":", $build->jobset->name, ':', $build->job->name, ") on ", $build->system, "\n";
edit in src/sql/hydra.sql at line 64

[10.1382]

[59.5479]

schedulingShares integer not null default 100,
file addition: upgrade-21.sql (----------)

[60.3004]

alter table Jobsets
add column schedulingShares integer not null default 100;