Store aggregate members in the database

[?]

Aug 13, 2013, 11:59 PM

FTPCV25MOLQUNR5CAR453W7T7QTUZRLPLEOSDZ5HSDFAXQZVHOYQC

Dependencies

[2] LZVO64YG Merge in the first bits of the API work
[3] PHNLYPKB Call buildFinished when a cached build is added
[4] FANTYCR7 Materialize the number of finished builds
[5] JOYONH2K Prevent multiple builds with the same (job, outPath) tuple from being added
[6] PMNWRTGJ Add multiple output support
[7] 3XTHEUMP * Implemented the clone feature.
[8] 5SHCWE7X * Prevent repeated evaluation of a jobset with the same inputs. This
[9] HPEG2RHV Merge the BuildResultInfo table into the Builds table
[10] QS4OX6Z7 Huuuge speedup in the Hydra evaluator
[11] MOX7XJ2E Merge the BuildSchedulingInfo table into the Builds table
[12] YFPZ46YK * hydra: added variant of build input type, 'build output (same system)' to allow better continous integration in one jobset for multiple system. it makes sure that the system of the build that is passed as input for a job has the same system as the job.
[13] RFE6T5LG * Store jobset evaluations in the database explicitly. This includes
[14] A22P7HCO hydra: at evaluation, check if path is already built, and mark as built in stead of adding to the queue.
[15] E7M2WP7A Remove unused Jobs columns
[16] FM4O2L4M hydra: if evaluator sees cached build, also add the buildproducts
[17] SB2V735V Keep track of the database schema version
[18] S6OISBQ3 * Mark the "current" builds in a jobset, i.e. those corresponding to
[19] G2T4WAHI Store the inputs of each evaluation in the database
[20] QMW24O5S Add support for Guile & Guix.
[21] Y6AHH4TH Remove the logfile and logSize columns from the database
[22] AZCCANUB Missing part of aaacf9eda36f5cdcc77cec20d6f49fbb6f925105
[23] KQS7DSKJ * Clean up indentation.
[24] L2E6EVE2 * Merged the Build and Job tables.
[25] JZE7DC2F Whitespace
[26] RXVJFQ5A Evaluator cleanups
[27] YTIDBFGU Drop unused "disabled" columns
[28] ZTQEU5QS Hydra: Add support for maxSilent meta attribute (also already added timeout, but not implemented the actual timeout for the build yet)
[29] SJLEZFC4 check getHydraPath in stead of Envvar HYDRA_DBI directly
[30] INNOEHO6 * Fix getBuildLog for bzip2'd files.
[31] CQTN62OH Die tabs die
[*] 4N5APGRG * Start of a helper tool to evaluate job expressions efficiently.
[*] HVXL2XUZ
[*] A63IHCMX * Register GC roots properly.
[*] LBNVQXUB * Build the /build stuff in a separate controller.
[*] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
[*] D5QIOJGP * Move everything up one directory.
[*] J5UVLXOK * Start of a basic Catalyst web interface.
[*] JM3DPYOM generated schema with new dbix class schema loader, grrrrrr
[*] AHTEIK7G * Added a maintainers field to the Builds table.
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

edit in src/c/hydra-eval-jobs.cc at line 163

[34.1270]

[35.126]

/* If this is an aggregate, then get its members. */
Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate"));
if (a != v.attrs->end() && state.forceBool(*a->value)) {
Bindings::iterator a = v.attrs->find(state.symbols.create("members"));
if (a == v.attrs->end())
throw EvalError("derivation must have a ‘members’ attribute");
PathSet context;
state.coerceToString(*a->value, context, true, false);
PathSet drvs;
foreach (PathSet::iterator, i, context)
if (i->at(0) == '!') {
size_t index = i->find("!", 1);
drvs.insert(string(*i, index + 1));
}
xmlAttrs["members"] = concatStringsSep(" ", drvs);
}
replacement in src/lib/Hydra/Controller/Build.pm at line 581

[5.973]→[5.973:1031](∅→∅)

$c->model('DB'), $build->project, $build->jobset,

[5.973]

[3.0]

$c->model('DB'), $build->jobset,
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 291

[5.5121]→[5.9116:9321](∅→∅),[5.9116]→[5.9116:9321](∅→∅)

if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) {
$validJob = 0;
}
}
if ($validJob) {
push(@filteredJobs, $job);

[5.5121]

[5.9321]

$validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system};
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 293

[5.9331]

[5.2088]

push(@filteredJobs, $job) if $validJob;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 389

[5.3507]→[3.123:248](∅→∅)

my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap, $plugins) = @_;

[5.3507]

[5.7099]

my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 410

[5.3957]→[5.3957:4039](∅→∅),[5.4074]→[5.4074:4090](∅→∅)

my $job = $jobset->jobs->update_or_create(
{ name => $jobName
});

[5.3898]

[5.4090]

my $job = $jobset->jobs->update_or_create({ name => $jobName });
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 431

[5.4387]→[5.7934:8023](∅→∅)

{ project => $project->name, jobset => $jobset->name, job => $job->name,

[5.4387]

[5.8023]

{ project => $jobset->project->name, jobset => $jobset->name, job => $jobName,
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 436

[5.793]→[5.793:842](∅→∅)

$buildIds->{$prevBuild->id} = 0;

[5.793]

[5.842]

$buildMap->{$prevBuild->id} = { new => 0, drvPath => $drvPath };
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 443

[5.279]→[5.8173:8247](∅→∅)

my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath};

[5.279]

[5.345]

my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath};
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 509

[5.5889]→[5.1505:1542](∅→∅),[5.1542]→[5.9179:9255](∅→∅)

$buildIds->{$build->id} = 1;
$$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id;

[5.5889]

[5.725]

$buildMap->{$build->id} = { new => 1, drvPath => $drvPath };
$$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;
file addition: AggregateMembers.pm (----------)

[38.477]

use utf8;
package Hydra::Schema::AggregateMembers;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Hydra::Schema::AggregateMembers

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 COMPONENTS LOADED

=over 4

=item * L<Hydra::Component::ToJSON>

=back

=cut

__PACKAGE__->load_components("+Hydra::Component::ToJSON");

=head1 TABLE: C<AggregateMembers>

=cut

__PACKAGE__->table("AggregateMembers");

=head1 ACCESSORS

=head2 aggregate

data_type: 'integer'
is_foreign_key: 1
is_nullable: 0

=head2 member

data_type: 'integer'
is_foreign_key: 1
is_nullable: 0

=cut

__PACKAGE__->add_columns(
"aggregate",
{ data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
"member",
{ data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
);

=head1 PRIMARY KEY

=over 4

=item * L</aggregate>

=item * L</member>

=back

=cut

__PACKAGE__->set_primary_key("aggregate", "member");

=head1 RELATIONS

=head2 aggregate

Type: belongs_to

Related object: L<Hydra::Schema::Builds>

=cut

__PACKAGE__->belongs_to(
"aggregate",
"Hydra::Schema::Builds",
{ id => "aggregate" },
{ is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);

=head2 member

Type: belongs_to

Related object: L<Hydra::Schema::Builds>

=cut

__PACKAGE__->belongs_to(
"member",
"Hydra::Schema::Builds",
{ id => "member" },
{ is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jHJtO2baXiprv0OcWCLZ+w

# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;
edit in src/lib/Hydra/Schema/Builds.pm at line 290

[40.8825]

[40.8825]

=head2 aggregatemembers_aggregates

Type: has_many

Related object: L<Hydra::Schema::AggregateMembers>

=cut

__PACKAGE__->has_many(
"aggregatemembers_aggregates",
"Hydra::Schema::AggregateMembers",
{ "foreign.aggregate" => "self.id" },
undef,
);

=head2 aggregatemembers_members

Type: has_many

Related object: L<Hydra::Schema::AggregateMembers>

=cut

__PACKAGE__->has_many(
"aggregatemembers_members",
"Hydra::Schema::AggregateMembers",
{ "foreign.member" => "self.id" },
undef,
);
edit in src/lib/Hydra/Schema/Builds.pm at line 500

[5.1180]

[5.62]

=head2 aggregates

Type: many_to_many

Composing rels: L</aggregatemembers_members> -> aggregate
edit in src/lib/Hydra/Schema/Builds.pm at line 507

[5.63]

[5.1181]

=cut
replacement in src/lib/Hydra/Schema/Builds.pm at line 509

[5.1182]→[2.28127:28269](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:isCEXACY/PwkvgKHcXvAIg

[5.1182]

[5.3377]

__PACKAGE__->many_to_many("aggregates", "aggregatemembers_members", "aggregate");

=head2 members

Type: many_to_many

Composing rels: L</aggregatemembers_members> -> member

=cut

__PACKAGE__->many_to_many("members", "aggregatemembers_members", "member");
edit in src/lib/Hydra/Schema/Builds.pm at line 522

[5.3378]

[41.9049]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9jqsol/evbHYjusT09hLtw
replacement in src/script/hydra-evaluator at line 147

[5.5628]→[5.2275:2297](∅→∅)

my %buildIds;

[5.5628]

[5.5655]

my %buildMap;
replacement in src/script/hydra-evaluator at line 151

[5.2411]→[3.911:1041](∅→∅)

checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap, $plugins);

[5.2411]

[5.5911]

checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
replacement in src/script/hydra-evaluator at line 165

[5.5964]→[5.2669:2719](∅→∅),[5.2719]→[5.6019:6058](∅→∅),[5.6019]→[5.6019:6058](∅→∅)

while (my ($id, $new) = each %buildIds) {
$hasNewBuilds = 1 if $new;

[5.5964]

[5.5839]

while (my ($id, $x) = each %buildMap) {
$hasNewBuilds = 1 if $x->{new};
replacement in src/script/hydra-evaluator at line 175

[5.6334]→[5.903:976](∅→∅)

, nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef

[5.6334]

[5.6334]

, nrbuilds => $hasNewBuilds ? scalar(keys %buildMap) : undef
replacement in src/script/hydra-evaluator at line 179

[5.6380]→[5.2720:2774](∅→∅),[5.2774]→[5.6439:6520](∅→∅),[5.6439]→[5.6439:6520](∅→∅)

while (my ($id, $new) = each %buildIds) {
$ev->jobsetevalmembers->create({ build => $id, isnew => $new });

[5.6380]

[5.3171]

# Create JobsetEvalMembers mappings.
my %drvPathToId;
while (my ($id, $x) = each %buildMap) {
$ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} });
$drvPathToId{$x->{drvPath}} = $id;
}

# Create AggregateMembers mappings.
foreach my $job (@{$jobs->{job}}) {
next unless $job->{members};
my $id = $drvPathToId{$job->{drvPath}} or die;
foreach my $drvPath (split / /, $job->{members}) {
my $member = $drvPathToId{$drvPath};
if (defined $member) {
$db->resultset('AggregateMembers')->update_or_create({aggregate => $id, member => $member});
} else {
warn "aggregate job ‘$job->{jobName}’ has a member ‘$drvPath’ that doesn't correspond to a Hydra build\n";
}
}
edit in src/sql/hydra.sql at line 517

[4.1368]

[4.1368]

create table AggregateMembers (
aggregate integer not null references Builds(id) on delete cascade,
member integer not null references Builds(id) on delete cascade,
primary key (aggregate, member)
);
file addition: upgrade-19.sql (----------)

[38.3004]

create table AggregateMembers (
aggregate integer not null references Builds(id) on delete cascade,
member integer not null references Builds(id) on delete cascade,
primary key (aggregate, member)
);