grahamc/hydra2 - Change T5BIOVJEMBIASP7EKQVV2N3VD6I56UXH6LCD5I33BDQEVHJAMGKQC

Add support for tracking custom metrics

Builds can now emit metrics that Hydra will store in its database and render as time series via flot charts. Typical applications are to keep track of performance indicators, coverage percentages, artifact sizes, and so on.

For example, a coverage build can emit the coverage percentage as follows:

echo "lineCoverage $pct %" > $out/nix-support/hydra-metrics

Graphs of all metrics for a job can be seen at

http://…/job/<project>/<jobset>/<job>#tabs-charts

Specific metrics are also visible at

http://…/job/<project>/<jobset>/<job>/metric/<metric>

The latter URL also allows getting the data in JSON format (e.g. via "curl -H 'Accept: application/json'").

Created by Eelco Dolstra on July 31, 2015

T5BIOVJEMBIASP7EKQVV2N3VD6I56UXH6LCD5I33BDQEVHJAMGKQC

Dependencies

In channels

main

Change contents

Insertion in src/hydra-queue-runner/build-result.cc at line 8 [10.441]

[10.553]


static std::tuple<bool, string> secureRead(Path fileName)
{
    auto fail = std::make_tuple(false, "");
    if (!pathExists(fileName)) return fail;
    try {
        /* For security, resolve symlinks. */
        fileName = canonPath(fileName, true);
        if (!isInStore(fileName)) return fail;
        return std::make_tuple(true, readFile(fileName));
    } catch (Error & e) { return fail; }
}

Replacement in src/hydra-queue-runner/build-result.cc at line 58 [10.441]

∅:D[5.114] → [10.1182:1305]

B:BD[10.1182] → [10.1182:1305]

        Path productsFile = output + "/nix-support/hydra-build-products";
        if (!pathExists(productsFile)) continue;

[5.114]

[10.1305]

        auto file = secureRead(output + "/nix-support/hydra-build-products");
        if (!std::get<0>(file)) continue;

Replacement in src/hydra-queue-runner/build-result.cc at line 63 [10.441]

B:BD[10.1339] → [10.1339:1385]

B:BD[10.1385] → [6.326:440]

∅:D[6.440] → [10.1439:1488]

B:BD[10.1439] → [10.1439:1488]

B:BD[10.1488] → [6.441:569]

∅:D[6.569] → [10.1524:1525]

B:BD[10.1524] → [10.1524:1525]

B:BD[10.1525] → [6.570:640]

        /* For security, resolve symlinks. */
        try {
            productsFile = canonPath(productsFile, true);
        } catch (Error & e) { continue; }
        if (!isInStore(productsFile)) continue;
        string contents;
        try {
            contents = readFile(productsFile);
        } catch (Error & e) { continue; }
        for (auto & line : tokenizeString<Strings>(contents, "\n")) {

[10.1339]

[6.640]

        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {

Insertion in src/hydra-queue-runner/build-result.cc at line 130 [10.441]

[10.4269]

    /* Get metrics. */
    for (auto & output : outputs) {
        auto file = secureRead(output + "/nix-support/hydra-metrics");
        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
            auto fields = tokenizeString<std::vector<std::string>>(line);
            if (fields.size() < 2) continue;
            BuildMetric metric;
            metric.name = fields[0]; // FIXME: validate
            metric.value = atof(fields[1].c_str()); // FIXME
            metric.unit = fields.size() >= 3 ? fields[2] : "";
            res.metrics[metric.name] = metric;
        }
    }

Insertion in src/hydra-queue-runner/build-result.hh at line 16 [10.4329]
[10.4593]
[10.4593]
```
};
struct BuildMetric
{
    std::string name, unit;
    double value;
```
Insertion in src/hydra-queue-runner/build-result.hh at line 36 [10.4329]
[10.4737]
[10.4737]
```
    std::map<std::string, BuildMetric> metrics;
```

Insertion in src/hydra-queue-runner/hydra-queue-runner.cc at line 241 [10.4840]

[10.19015]

[11.410]

    for (auto & metric : res.metrics) {
        txn.parameterized
            ("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)")
            (build->id)
            (metric.second.name)
            (metric.second.unit, metric.second.unit != "")
            (metric.second.value)
            (build->projectName)
            (build->jobsetName)
            (build->jobName)
            (build->timestamp).exec();
    }

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 67 [8.20067]

B:BD[8.21931] → [8.21931:22108]

        auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();

[8.21931]

[8.22108]

        auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 79 [8.20067]

B:BD[8.22529] → [8.22529:22658]

            build->fullJobName = row["project"].as<string>() + ":" + row["jobset"].as<string>() + ":" + row["job"].as<string>();

[8.22529]

[8.22658]

            build->projectName = row["project"].as<string>();
            build->jobsetName = row["jobset"].as<string>();
            build->jobName = row["job"].as<string>();

Insertion in src/hydra-queue-runner/queue-monitor.cc at line 84 [8.20067]
[8.22781]
[8.22781]
```
            build->timestamp = row["timestamp"].as<time_t>();
```

Replacement in src/hydra-queue-runner/queue-monitor.cc at line 95 [8.20067]

B:BD[8.23024] → [8.23024:23124]

        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName);

[8.23024]

[8.23124]

        printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName());

Insertion in src/hydra-queue-runner/state.hh at line 41 [12.1128]
[12.1843]
[12.1843]
```
    bssCachedFailure = 8,
```
Replacement in src/hydra-queue-runner/state.hh at line 71 [12.1128]
∅:D[7.354] → [12.2121:2150]
B:BD[12.2121] → [12.2121:2150]
```
    std::string fullJobName;
```
[7.354]
[12.2150]
```
    std::string projectName, jobsetName, jobName;
    time_t timestamp;
```

Insertion in src/hydra-queue-runner/state.hh at line 78 [12.1128]

[12.2276]


    std::string fullJobName()
    {
        return projectName + ":" + jobsetName + ":" + jobName;
    }

Insertion in src/lib/Hydra/Controller/Job.pm at line 80 [14.46]

[13.228]

[4.0]


    $c->stash->{metrics} = [ $job->buildmetrics->search(
        { }, { select => ["name"], distinct => 1, order_by => "timestamp desc",  }) ];

Insertion in src/lib/Hydra/Controller/Job.pm at line 116 [14.46]

[15.404]

[14.548]

sub metric : Chained('job') PathPart('metric') Args(1) {
    my ($self, $c, $metricName) = @_;
    $c->stash->{template} = 'metric.tt';
    $c->stash->{metricName} = $metricName;
    my @res = $c->stash->{job}->buildmetrics->search(
        { name => $metricName },
        { order_by => "timestamp", columns => [ "build", "name", "timestamp", "value", "unit" ] });
    $self->status_ok($c, entity => [ map { { id => $_->get_column("build"), timestamp => $_ ->timestamp, value => $_->value, unit => $_->unit } } @res ]);
}

File addition: BuildMetrics.pm (----------)

[16.477]

use utf8;
package Hydra::Schema::BuildMetrics;
# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE
=head1 NAME
Hydra::Schema::BuildMetrics
=cut
use strict;
use warnings;
use base 'DBIx::Class::Core';
=head1 COMPONENTS LOADED
=over 4
=item * L<Hydra::Component::ToJSON>
=back
=cut
__PACKAGE__->load_components("+Hydra::Component::ToJSON");
=head1 TABLE: C<BuildMetrics>
=cut
__PACKAGE__->table("BuildMetrics");
=head1 ACCESSORS
=head2 build
  data_type: 'integer'
  is_foreign_key: 1
  is_nullable: 0
=head2 name
  data_type: 'text'
  is_nullable: 0
=head2 unit
  data_type: 'text'
  is_nullable: 1
=head2 value
  data_type: 'double precision'
  is_nullable: 0
=head2 project
  data_type: 'text'
  is_foreign_key: 1
  is_nullable: 0
=head2 jobset
  data_type: 'text'
  is_foreign_key: 1
  is_nullable: 0
=head2 job
  data_type: 'text'
  is_foreign_key: 1
  is_nullable: 0
=head2 timestamp
  data_type: 'integer'
  is_nullable: 0
=cut
__PACKAGE__->add_columns(
  "build",
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
  "name",
  { data_type => "text", is_nullable => 0 },
  "unit",
  { data_type => "text", is_nullable => 1 },
  "value",
  { data_type => "double precision", is_nullable => 0 },
  "project",
  { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
  "jobset",
  { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
  "job",
  { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
  "timestamp",
  { data_type => "integer", is_nullable => 0 },
);
=head1 PRIMARY KEY
=over 4
=item * L</build>
=item * L</name>
=back
=cut
__PACKAGE__->set_primary_key("build", "name");
=head1 RELATIONS
=head2 build
Type: belongs_to
Related object: L<Hydra::Schema::Builds>
=cut
__PACKAGE__->belongs_to(
  "build",
  "Hydra::Schema::Builds",
  { id => "build" },
  { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
);
=head2 job
Type: belongs_to
Related object: L<Hydra::Schema::Jobs>
=cut
__PACKAGE__->belongs_to(
  "job",
  "Hydra::Schema::Jobs",
  { jobset => "jobset", name => "job", project => "project" },
  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" },
);
=head2 jobset
Type: belongs_to
Related object: L<Hydra::Schema::Jobsets>
=cut
__PACKAGE__->belongs_to(
  "jobset",
  "Hydra::Schema::Jobsets",
  { name => "jobset", project => "project" },
  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" },
);
=head2 project
Type: belongs_to
Related object: L<Hydra::Schema::Projects>
=cut
__PACKAGE__->belongs_to(
  "project",
  "Hydra::Schema::Projects",
  { name => "project" },
  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" },
);
# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qoPm5/le+sVHigW4Dmum2Q
sub json_hint {
    return { columns => ['value', 'unit'] };
}
1;

Insertion in src/lib/Hydra/Schema/Builds.pm at line 341 [19.5202]

[17.9620]

[18.10120]

  undef,
);
=head2 buildmetrics
Type: has_many
Related object: L<Hydra::Schema::BuildMetrics>
=cut
__PACKAGE__->has_many(
  "buildmetrics",
  "Hydra::Schema::BuildMetrics",
  { "foreign.build" => "self.id" },

Replacement in src/lib/Hydra/Schema/Builds.pm at line 553 [19.5202]

B:BD[20.3378] → [9.357:499]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:03:55
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EwxiaQpqbdzI9RvU0uUtLQ

[20.3378]

[21.4481]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y2lDtgY8EBLOuCHAI8fWRQ

Insertion in src/lib/Hydra/Schema/Builds.pm at line 648 [19.5202]
[2.502]
[22.158]
```
        buildmetrics => 'name',
```

Insertion in src/lib/Hydra/Schema/Jobs.pm at line 83 [24.1454]

[23.14424]


=head2 buildmetrics
Type: has_many
Related object: L<Hydra::Schema::BuildMetrics>
=cut

Insertion in src/lib/Hydra/Schema/Jobs.pm at line 92 [24.1454]

[23.14425]

[17.15942]

__PACKAGE__->has_many(
  "buildmetrics",
  "Hydra::Schema::BuildMetrics",
  {
    "foreign.job"     => "self.name",
    "foreign.jobset"  => "self.jobset",
    "foreign.project" => "self.project",
  },
  undef,
);

Replacement in src/lib/Hydra/Schema/Jobs.pm at line 172 [24.1454]

B:BD[13.1556] → [25.0:142]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-09-29 19:41:42
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lnZSd0gDXgLk8WQeAFqByA

[13.1556]

[24.2756]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vDAo9bzLca+QWfhOb9OLMg

Insertion in src/lib/Hydra/Schema/Jobsets.pm at line 186 [26.1960]

[17.21696]

[23.19217]


=head2 buildmetrics
Type: has_many
Related object: L<Hydra::Schema::BuildMetrics>
=cut
__PACKAGE__->has_many(
  "buildmetrics",
  "Hydra::Schema::BuildMetrics",
  {
    "foreign.jobset"  => "self.name",
    "foreign.project" => "self.project",
  },
  undef,
);

Replacement in src/lib/Hydra/Schema/Jobsets.pm at line 341 [26.1960]

B:BD[17.22628] → [3.0:142]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:51
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CO0aE+jrjB+UrwGRzWZLlw

[17.22628]

[25.217]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Coci9FdBAvUO9T3st2NEqA

Insertion in src/lib/Hydra/Schema/Projects.pm at line 108 [26.3266]
[17.24478]
[27.4154]
```
=head2 buildmetrics
Type: has_many
Related object: L<Hydra::Schema::BuildMetrics>
```

Insertion in src/lib/Hydra/Schema/Projects.pm at line 115 [26.3266]

[27.4155]

[17.24479]

=cut
__PACKAGE__->has_many(
  "buildmetrics",
  "Hydra::Schema::BuildMetrics",
  { "foreign.project" => "self.name" },
  undef,
);

Replacement in src/lib/Hydra/Schema/Projects.pm at line 285 [26.3266]

B:BD[28.7308] → [3.173:315]

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:08
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fkd9ruEoVSBGIktmAj4u4g

[28.7308]

[29.171]

# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw

Replacement in src/root/build.tt at line 109 [19.7350]
B:BD[30.1217] → [31.0:3]
```
  
```
[30.1217]
[32.1530]

Insertion in src/root/build.tt at line 388 [19.7350]

[32.10758]


    [% IF build.finished && build.buildmetrics %]
      <h3>Metrics</h3>
      <table class="table table-small table-striped table-hover clickable-rows">
        <thead>
          <tr><th>Name</th><th>Value</th></tr>
        </thead>
        <tbody>
          [% FOREACH metric IN build.buildmetrics %]
            <tr>
              <td><tt><a class="row-link" href="[% c.uri_for('/job' project.name jobset.name job.name 'metric' metric.name) %]">[%HTML.escape(metric.name)%]</a></tt></td>
              <td>[%metric.value%][%metric.unit%]</td>
            </tr>
          [% END %]
        </tbody>
      </table>
    [% END %]

Replacement in src/root/common.tt at line 569 [33.2803]

B:BD[15.809] → [15.809:832]

          var max = 0;

[15.809]

[15.832]

          var maxTime = 0;
          var minTime = Number.MAX_SAFE_INTEGER;

Replacement in src/root/common.tt at line 575 [33.2803]

∅:D[4.636] → [15.990:1026]

B:BD[15.990] → [15.990:1026]

            max = Math.max(t, max);

[4.636]

[15.1026]

            maxTime = Math.max(t, maxTime);
            minTime = Math.min(t, minTime);

Replacement in src/root/common.tt at line 639 [33.2803]

B:BD[15.2763] → [15.2763:2854]

          plot.setSelection({ xaxis: { from: max - 60 * 24 * 60 * 60 * 1000, to: max } });

[15.2763]

[15.2854]

          plot.setSelection({ xaxis: { from: Math.max(minTime, maxTime - 60 * 24 * 60 * 60 * 1000), to: maxTime } });

Insertion in src/root/job.tt at line 100 [35.7675]

[4.1081]

[34.3469]


    [% FOREACH metric IN metrics %]
      <h3>Metric: <tt>[%HTML.escape(metric.name)%]</tt></h3>
      [% INCLUDE createChart id="metric-${metric.name}" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'metric' metric.name) %]
    [% END %]

File addition: metric.tt (----------)

[16.1486]

[% WRAPPER layout.tt title="Job metric ‘$metricName’" %]
[% PROCESS common.tt %]
[% INCLUDE includeFlot %]
[% INCLUDE createChart id="chart" dataUrl=c.req.uri %]
[% END %]

Insertion in src/root/static/css/hydra.css at line 30 [19.8691]
[36.81]
[36.81]
```
}
table.table-small {
    width: auto !important;
```

Insertion in src/sql/hydra.sql at line 325 [37.1]

[37.1028]


create table BuildMetrics (
    build         integer not null,
    name          text not null,

Insertion in src/sql/hydra.sql at line 331 [37.1]

[37.1029]

    unit          text,
    value         double precision not null,

Insertion in src/sql/hydra.sql at line 334 [37.1]

[37.1030]

[38.4327]

    -- Denormalisation for performance: copy some columns from the
    -- corresponding build.
    project       text not null,
    jobset        text not null,
    job           text not null,
    timestamp     integer not null,
    primary key   (build, name),
    foreign key   (build) references Builds(id) on delete cascade,
    foreign key   (project) references Projects(name) on update cascade,
    foreign key   (project, jobset) references Jobsets(project, name) on update cascade,
    foreign key   (project, jobset, job) references Jobs(project, jobset, name) on update cascade
);

Insertion in src/sql/hydra.sql at line 615 [37.1]

[39.2243]

create index IndexBuildMetricsOnJobTimestamp on BuildMetrics(project, jobset, job, timestamp desc);

File addition: upgrade-39.sql (----------)

[16.3004]

create table BuildMetrics (
    build         integer not null,
    name          text not null,
    unit          text,
    value         double precision not null,
    -- Denormalisation for performance: copy some columns from the
    -- corresponding build.
    project       text not null,
    jobset        text not null,
    job           text not null,
    timestamp     integer not null,
    primary key   (build, name),
    foreign key   (build) references Builds(id) on delete cascade,
    foreign key   (project) references Projects(name) on update cascade,
    foreign key   (project, jobset) references Jobsets(project, name) on update cascade,
    foreign key   (project, jobset, job) references Jobs(project, jobset, name) on update cascade
);
create index IndexBuildMetricsOnJobTimestamp on BuildMetrics(project, jobset, job, timestamp desc);