grahamc/hydra - Change FTPCV25MOLQUNR5CAR453W7T7QTUZRLPLEOSDZ5HSDFAXQZVHOYQC

Store aggregate members in the database

For presentation purposes, we need to know what builds are part of an aggregate build. So at evaluation time, look at the "members" attribute, find the corresponding builds in the eval, and create a mapping in the AggregateMembers table.

Created by Eelco Dolstra on August 13, 2013

FTPCV25MOLQUNR5CAR453W7T7QTUZRLPLEOSDZ5HSDFAXQZVHOYQC

Dependencies

In channels

main

upstream

Change contents

Insertion in src/c/hydra-eval-jobs.cc at line 163 [7.303]

[5.1270]

[6.126]

            /* If this is an aggregate, then get its members. */
            Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate"));
            if (a != v.attrs->end() && state.forceBool(*a->value)) {
                Bindings::iterator a = v.attrs->find(state.symbols.create("members"));
                if (a == v.attrs->end())
                    throw EvalError("derivation must have a ‘members’ attribute");
                PathSet context;
                state.coerceToString(*a->value, context, true, false);
                PathSet drvs;
                foreach (PathSet::iterator, i, context)
                    if (i->at(0) == '!') {
                        size_t index = i->find("!", 1);
                        drvs.insert(string(*i, index + 1));
                    }
                xmlAttrs["members"] = concatStringsSep(" ", drvs);
            }

Replacement in src/lib/Hydra/Controller/Build.pm at line 581 [9.1]

B:BD[8.973] → [8.973:1031]

        $c->model('DB'), $build->project, $build->jobset,

[8.973]

[3.0]

        $c->model('DB'), $build->jobset,

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 291 [12.2663]

∅:D[10.5121] → [11.9116:9321]

B:BD[11.9116] → [11.9116:9321]

            if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) {
                $validJob = 0;
            }
        }
        if ($validJob) {
            push(@filteredJobs, $job);

[10.5121]

[11.9321]

            $validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system};

Insertion in src/lib/Hydra/Helper/AddBuilds.pm at line 293 [12.2663]
[11.9331]
[13.2088]
```
        push(@filteredJobs, $job) if $validJob;
```

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 389 [12.2663]

B:BD[8.3507] → [3.123:248]

    my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap, $plugins) = @_;

[8.3507]

[10.7099]

    my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 410 [12.2663]

B:BD[8.3957] → [8.3957:4039]

B:BD[8.4074] → [8.4074:4090]

        my $job = $jobset->jobs->update_or_create(
            { name => $jobName
            });

[8.3898]

[8.4090]

        my $job = $jobset->jobs->update_or_create({ name => $jobName });

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 431 [12.2663]

B:BD[14.4387] → [10.7934:8023]

                { project => $project->name, jobset => $jobset->name, job => $job->name,

[14.4387]

[10.8023]

                { project => $jobset->project->name, jobset => $jobset->name, job => $jobName,

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 436 [12.2663]

B:BD[15.793] → [15.793:842]

                $buildIds->{$prevBuild->id} = 0;

[15.793]

[15.842]

                $buildMap->{$prevBuild->id} = { new => 0, drvPath => $drvPath };

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 443 [12.2663]

B:BD[16.279] → [10.8173:8247]

        my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath};

[16.279]

[16.345]

        my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath};

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 509 [12.2663]

B:BD[8.5889] → [15.1505:1542]

B:BD[15.1542] → [10.9179:9255]

        $buildIds->{$build->id} = 1;
        $$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id;

[8.5889]

[17.725]

        $buildMap->{$build->id} = { new => 1, drvPath => $drvPath };
        $$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id;

File addition: AggregateMembers.pm (----------)

[18.477]

use utf8;
package Hydra::Schema::AggregateMembers;
# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE
=head1 NAME
Hydra::Schema::AggregateMembers
=cut
use strict;
use warnings;
use base 'DBIx::Class::Core';
=head1 COMPONENTS LOADED
=over 4
=item * L<Hydra::Component::ToJSON>
=back
=cut
__PACKAGE__->load_components("+Hydra::Component::ToJSON");
=head1 TABLE: C<AggregateMembers>
=cut
__PACKAGE__->table("AggregateMembers");
=head1 ACCESSORS
=head2 aggregate
  data_type: 'integer'
  is_foreign_key: 1
  is_nullable: 0
=head2 member
  data_type: 'integer'
  is_foreign_key: 1
  is_nullable: 0
=cut
__PACKAGE__->add_columns(
  "aggregate",
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
  "member",
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
);
=head1 PRIMARY KEY
=over 4
=item * L</aggregate>
=item * L</member>
=back
=cut
__PACKAGE__->set_primary_key("aggregate", "member");
=head1 RELATIONS
=head2 aggregate
Type: belongs_to
Related object: L<Hydra::Schema::Builds>
=cut
__PACKAGE__->belongs_to(
  "aggregate",
  "Hydra::Schema::Builds",
  { id => "aggregate" },
  { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);
=head2 member
Type: belongs_to
Related object: L<Hydra::Schema::Builds>
=cut
__PACKAGE__->belongs_to(
  "member",
  "Hydra::Schema::Builds",
  { id => "member" },
  { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jHJtO2baXiprv0OcWCLZ+w
# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;

Insertion in src/lib/Hydra/Schema/Builds.pm at line 290 [20.5202]

[19.8825]


=head2 aggregatemembers_aggregates
Type: has_many
Related object: L<Hydra::Schema::AggregateMembers>
=cut
__PACKAGE__->has_many(
  "aggregatemembers_aggregates",
  "Hydra::Schema::AggregateMembers",
  { "foreign.aggregate" => "self.id" },
  undef,
);
=head2 aggregatemembers_members
Type: has_many
Related object: L<Hydra::Schema::AggregateMembers>
=cut
__PACKAGE__->has_many(
  "aggregatemembers_members",
  "Hydra::Schema::AggregateMembers",
  { "foreign.member" => "self.id" },
  undef,
);

Insertion in src/lib/Hydra/Schema/Builds.pm at line 500 [20.5202]

[21.1180]

[22.62]


=head2 aggregates
Type: many_to_many
Composing rels: L</aggregatemembers_members> -> aggregate

Insertion in src/lib/Hydra/Schema/Builds.pm at line 507 [20.5202]
[22.63]
[21.1181]
```
=cut
```

Replacement in src/lib/Hydra/Schema/Builds.pm at line 509 [20.5202]

B:BD[21.1182] → [2.28127:28269]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:isCEXACY/PwkvgKHcXvAIg

[21.1182]

[23.3377]

__PACKAGE__->many_to_many("aggregates", "aggregatemembers_members", "aggregate");
=head2 members
Type: many_to_many
Composing rels: L</aggregatemembers_members> -> member
=cut
__PACKAGE__->many_to_many("members", "aggregatemembers_members", "member");

Insertion in src/lib/Hydra/Schema/Builds.pm at line 522 [20.5202]

[23.3378]

[24.9049]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9jqsol/evbHYjusT09hLtw

Replacement in src/script/hydra-evaluator at line 147 [25.1319]
B:BD[21.5628] → [15.2275:2297]
```
        my %buildIds;
```
[21.5628]
[21.5655]
```
        my %buildMap;
```

Replacement in src/script/hydra-evaluator at line 151 [25.1319]

B:BD[15.2411] → [3.911:1041]

            checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap, $plugins);

[15.2411]

[21.5911]

            checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);

Replacement in src/script/hydra-evaluator at line 165 [25.1319]

B:BD[21.5964] → [15.2669:2719]

∅:D[15.2719] → [21.6019:6058]

B:BD[21.6019] → [21.6019:6058]

        while (my ($id, $new) = each %buildIds) {
            $hasNewBuilds = 1 if $new;

[21.5964]

[26.5839]

        while (my ($id, $x) = each %buildMap) {
            $hasNewBuilds = 1 if $x->{new};

Replacement in src/script/hydra-evaluator at line 175 [25.1319]

B:BD[21.6334] → [27.903:976]

            , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef

[21.6334]

            , nrbuilds => $hasNewBuilds ? scalar(keys %buildMap) : undef

Replacement in src/script/hydra-evaluator at line 179 [25.1319]

B:BD[21.6380] → [15.2720:2774]

∅:D[15.2774] → [21.6439:6520]

B:BD[21.6439] → [21.6439:6520]

            while (my ($id, $new) = each %buildIds) {
                $ev->jobsetevalmembers->create({ build => $id, isnew => $new });

[21.6380]

[28.3171]

            # Create JobsetEvalMembers mappings.
            my %drvPathToId;
            while (my ($id, $x) = each %buildMap) {
                $ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} });
                $drvPathToId{$x->{drvPath}} = $id;
            }
            # Create AggregateMembers mappings.
            foreach my $job (@{$jobs->{job}}) {
                next unless $job->{members};
                my $id = $drvPathToId{$job->{drvPath}} or die;
                foreach my $drvPath (split / /, $job->{members}) {
                    my $member = $drvPathToId{$drvPath};
                    if (defined $member) {
                        $db->resultset('AggregateMembers')->update_or_create({aggregate => $id, member => $member});
                    } else {
                        warn "aggregate job ‘$job->{jobName}’ has a member ‘$drvPath’ that doesn't correspond to a Hydra build\n";
                    }
                }

Insertion in src/sql/hydra.sql at line 517 [25.1]

[4.1368]

create table AggregateMembers (
    aggregate     integer not null references Builds(id) on delete cascade,
    member        integer not null references Builds(id) on delete cascade,
    primary key   (aggregate, member)
);

File addition: upgrade-19.sql (----------)

[18.3004]

create table AggregateMembers (
    aggregate     integer not null references Builds(id) on delete cascade,
    member        integer not null references Builds(id) on delete cascade,
    primary key   (aggregate, member)
);