Keep track of failed paths in the Hydra database

[?]

Jun 10, 2015, 12:57 PM

RYTQLATYOZ6ODIKYVJ63TC4OIQBXHSCV3NA2YD4NFP7443GQVSRQC

Dependencies

[2] VHV6GI4L Add a jobset eval action to restart all aborted/cancelled builds
[3] KSBB33RE Add a dashboard
[4] 6SJQECSC Fix broken redirects
[5] LJILHOJ7 Create BuildSteps race-free
[6] ZWCTAZGL added newsitems, added some admin options to clear various caches.
[7] NJJ7H64S Very basic multi-threaded queue runner
[8] 24BMQDZA Start of single-process hydra-queue-runner
[9] YZAI5GQU Implement a database connection pool
[10] KBZHIGLG Record the machine used for a build step
[11] J5ITV54P Make restartBuilds faster
[12] FQQRJUO4 Mark builds as busy
[13] 5NSQUYBS Clear failed builds etc.: Redirect back to the referrer
[14] ENXUSMSV Make concurrency more robust
[15] UOINKJ2J Add an action to cancel all builds in a jobset eval
[16] 5AIYUMTB Basic remote building
[*] 2GK5DOU7 * Downloading closures.
[*] D5QIOJGP * Move everything up one directory.
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 783

[6.12803]→[6.12803:12901](∅→∅)

printMsg(lvlError, format("build thread for ‘%1%’: %2%") % step->drvPath % e.what());

[6.12803]

[6.12901]

printMsg(lvlError, format("error building ‘%1%’: %2%") % step->drvPath % e.what());
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 831

[6.15416]→[6.15416:15477](∅→∅),[6.15477]→[6.395:454](∅→∅)

/* Create a build step record indicating that we started
building. Also, mark the selected build as busy. */
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 832

[6.707]

[6.13243]
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 834

[6.13268]

[6.13268]

BuildResult res;
int stepNr = 0;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 838

[6.13300]→[6.15550:15566](∅→∅),[6.15550]→[6.15550:15566](∅→∅)

int stepNr;

[6.13300]

[6.15566]

/* If any of the outputs have previously failed, then don't
retry. */
bool cachedFailure = false;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 844

[6.739]→[6.1015:1112](∅→∅),[6.1112]→[6.456:545](∅→∅),[6.456]→[6.456:545](∅→∅),[6.546]→[6.15674:15696](∅→∅),[6.13380]→[6.15674:15696](∅→∅),[6.15674]→[6.15674:15696](∅→∅)

stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bssBusy);
txn.parameterized("update Builds set busy = 1 where id = $1")(build->id).exec();
txn.commit();

[6.739]

[6.15696]

for (auto & path : outputPaths(step->drv))
if (!txn.parameterized("select 1 from FailedPaths where path = $1")(path).exec().empty()) {
cachedFailure = true;
break;
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 851

[6.15755]→[6.15755:15765](∅→∅),[6.15765]→[6.13381:13486](∅→∅),[6.13486]→[6.15842:15868](∅→∅),[6.15842]→[6.15842:15868](∅→∅),[6.15868]→[6.13487:13652](∅→∅),[6.13652]→[6.15896:15902](∅→∅),[6.15896]→[6.15896:15902](∅→∅)

try {
buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, logDir, result);
} catch (Error & e) {
result.status = RemoteResult::rrMiscFailure;
result.errorMsg = e.msg();
printMsg(lvlError, format("ERROR: %1%") % e.msg());
abort();
}

[6.15703]

[6.15902]

if (cachedFailure)
result.status = RemoteResult::rrPermanentFailure;
else {
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 855

[6.15903]→[6.13653:13706](∅→∅)

if (!result.stopTime) result.stopTime = time(0);

[6.15903]

[6.15934]

/* Create a build step record indicating that we started
building. Also, mark the selected build as busy. */
{
pqxx::work txn(*conn);
stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bssBusy);
txn.parameterized("update Builds set busy = 1 where id = $1")(build->id).exec();
txn.commit();
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 864

[6.15935]→[6.15935:15956](∅→∅),[6.15956]→[6.13707:13797](∅→∅)

BuildResult res;
if (result.status == RemoteResult::rrSuccess) res = getBuildResult(store, step->drv);

[6.15935]

[6.16006]

try {
buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, logDir, result);
} catch (Error & e) {
result.status = RemoteResult::rrMiscFailure;
result.errorMsg = e.msg();
printMsg(lvlError, format("ERROR: %1%") % e.msg());
abort();
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 873

[6.16007]→[6.16007:16047](∅→∅)

// FIXME: handle failed-with-output

[6.16007]

[6.16047]

if (result.status == RemoteResult::rrSuccess) res = getBuildResult(store, step->drv);

// FIXME: handle failed-with-output
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 878

[6.16048]

[6.8142]

if (!result.stopTime) result.stopTime = time(0);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 917

[6.16605]→[6.16605:16702](∅→∅),[6.16787]→[6.9048:9093](∅→∅),[6.9093]→[6.16828:16875](∅→∅),[6.16828]→[6.16828:16875](∅→∅),[6.16875]→[6.1367:1493](∅→∅),[6.1493]→[6.16969:16983](∅→∅),[6.14278]→[6.16969:16983](∅→∅),[6.16969]→[6.16969:16983](∅→∅)

/* Create failed build steps for every build that depends
on this. */
for (auto build2 : dependents) {
if (build == build2) continue;
createBuildStep(txn, result.stopTime, build2, step, machine->sshName, bssFailed, result.errorMsg, build->id);
}

[6.16605]

[5.199]

/* Failure case. */
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 919

[5.200]→[5.200:334](∅→∅)

finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, machine->sshName, bssFailed, result.errorMsg);

[5.200]

[6.16983]

/* For regular failures, we don't care about the error
message. */
if (result.status != RemoteResult::rrMiscFailure) result.errorMsg = "";

if (!cachedFailure) {

/* Create failed build steps for every build that depends
on this. */
for (auto build2 : dependents) {
if (build == build2) continue;
createBuildStep(txn, result.stopTime, build2, step, machine->sshName, bssFailed, result.errorMsg, build->id);
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 932

[6.16984]

[6.16984]

finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, machine->sshName, bssFailed, result.errorMsg);
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 939

[6.17135]→[6.547:694](∅→∅)

("update Builds set finished = 1, busy = 0, isCachedBuild = 0, buildStatus = $2, startTime = $3, stopTime = $4 where id = $1")

[6.17135]

[6.17272]

("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 943

[6.14407]→[6.14407:14453](∅→∅)

(result.stopTime).exec();

[6.14407]

[6.17464]

(result.stopTime)
(cachedFailure ? 1 : 0).exec();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 947

[6.17548]

[6.17548]

/* Remember failed paths in the database so that they
won't be built again. */
if (!cachedFailure && result.status == RemoteResult::rrPermanentFailure)
for (auto & path : outputPaths(step->drv))
txn.parameterized("insert into FailedPaths values ($1)")(path).exec();
replacement in src/lib/Hydra/Controller/Admin.pm at line 48

[6.398]→[6.398:448](∅→∅)

my $r = `nix-store --clear-failed-paths '*'`;

[6.375]

[4.53]

$c->model('DB::FailedPaths')->delete;
replacement in src/lib/Hydra/Helper/Nix.pm at line 468

[2.1493]→[2.1493:1539](∅→∅)

# Clear Nix's negative failure cache.

[2.1493]

[2.1539]

# Clear the failed paths cache.
replacement in src/lib/Hydra/Helper/Nix.pm at line 470

[2.1577]→[2.1577:1638](∅→∅)

system("nix-store", "--clear-failed-paths", @paths);

[2.1577]

[6.1608]

# FIXME: clear the dependencies?
$db->resultset('FailedPaths')->search({ path => [ @paths ]})->delete;
file addition: FailedPaths.pm (----------)

[19.477]

use utf8;
package Hydra::Schema::FailedPaths;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Hydra::Schema::FailedPaths

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 COMPONENTS LOADED

=over 4

=item * L<Hydra::Component::ToJSON>

=back

=cut

__PACKAGE__->load_components("+Hydra::Component::ToJSON");

=head1 TABLE: C<FailedPaths>

=cut

__PACKAGE__->table("FailedPaths");

=head1 ACCESSORS

=head2 path

data_type: 'text'
is_nullable: 0

=cut

__PACKAGE__->add_columns("path", { data_type => "text", is_nullable => 0 });

=head1 PRIMARY KEY

=over 4

=item * L</path>

=back

=cut

__PACKAGE__->set_primary_key("path");

# Created by DBIx::Class::Schema::Loader v0.07033 @ 2015-06-10 14:48:16
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:WFgjfjH+szE6Ntcicmaflw

# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;
edit in src/sql/hydra.sql at line 511

[3.8498]

[3.8498]

);

-- The output paths that have permanently failed.
create table FailedPaths (
path text primary key not null
edit in src/sql/hydra.sql at line 518

[3.8501]

[3.8501]

#ifdef POSTGRESQL

-- Needed because Postgres doesn't have "ignore duplicate" or upsert
-- yet.
create rule IdempotentInsert as on insert to FailedPaths
where exists (select 1 from FailedPaths where path = new.path)
do instead nothing;

#endif