Speed up channel processing

[?]

Mar 8, 2012, 12:17 AM

HXBXDEFK2VLXZIWUXAQ765NHKYQ5FYS43IKMU2DJ755L26TQ6IFQC

Dependencies

[2] HGVQUIUX * hydra: download closure now makes closure of outpath in stead of useless drvpath
[3] JO6L3MIJ * Include the system type in manifests as a hint for the
[4] 7G5RWHB7 Remove the "all" channel
[5] N45RZUQ6 Reduce I/O in build listings by only fetching required columns
[6] LBNVQXUB * Build the /build stuff in a separate controller.
[7] 2I2ZX6JB * Make the "latest succeeded" query (used by the "latest" channel)
[8] 7UJ5YV4V * Provide a channel for each project containing all the latest
[9] TPDJL4TY * Add derivations and all build-time dependencies of a build to the
[10] 2JQ7WEFM
[11] BHZXGT2H * Channels: provide an index page that lists all the packages in the
[12] 5NO7NCKT * Refactoring.
[13] XHVZXX6N
[14] BPVJBR2P * Only put Nix builds in the global channel.
[15] CS7T2XFI
[16] 7DWCXNC7 Use the new Nix Perl bindings
[17] SJN2QPWH * Big speed-up of the job status page and the channel generation (such
[18] A52HEFHQ * Allow builds to be restarted (if they failed with a transient error,
[19] FPK5LF53 * Put the project-related actions in a separate controller. Put the
[20] FM4O2L4M hydra: if evaluator sees cached build, also add the buildproducts
[21] TCHEWSZL * The "latest" channel shouldn't contain inactive jobs.
[22] KAZWI5G4 * hydra: buildpage, show changes since last build/successful build
[23] HPEG2RHV Merge the BuildResultInfo table into the Builds table
[24] Z4KRJX4Q * In the last succeeded / job status queries, use the Builds.isCurrent
[25] UXVDOUBJ
[26] RI4S7SYT * Job status: show the active jobs.
[27] AKAZKCR6 * At top-level and for each project, provide two channels: "latest"
[*] MVB7RRLT * Move NARs from the NixChannel controller to the Root controller and

Change contents

replacement in src/lib/Hydra/Base/Controller/ListBuilds.pm at line 79

[6.289]→[6.421:491](∅→∅)

getChannelData($c, scalar($c->stash->{latestSucceeded}));

[6.289]

[6.577]

$c->stash->{channelBuilds} = $c->stash->{latestSucceeded}
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });
edit in src/lib/Hydra/Base/Controller/ListBuilds.pm at line 83

[6.587]→[4.0:196](∅→∅)

#elsif ($channelName eq "all") {
# $c->stash->{channelName} = $c->stash->{channelBaseName} . "-all";
# getChannelData($c, scalar($c->stash->{allBuilds}));
#}
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 6

[6.1060]

[6.144]

use Nix::Store;
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 9

[6.202]

[6.202]
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 11

[6.203]

[6.203]

sub getChannelData {
my ($c, $checkValidity) = @_;

my @storePaths = ();
foreach my $build ($c->stash->{channelBuilds}->all) {
next if $checkValidity && !isValidPath($build->outpath);
#if (isValidPath($build->drvpath)) {
# # Adding `drvpath' implies adding `outpath' because of the
# # `--include-outputs' flag passed to `nix-store'.
# push @storePaths, $build->drvpath;
#} else {
# push @storePaths, $build->outpath;
#}
push @storePaths, $build->outpath;
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
$c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
# Put the system type in the manifest (for top-level paths) as
# a hint to the binary patch generator. (It shouldn't try to
# generate patches between builds for different systems.) It
# would be nice if Nix stored this info for every path but it
# doesn't.
$c->stash->{systemForPath}->{$build->outpath} = $build->system;
};
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 35

[6.204]

[6.204]

$c->stash->{storePaths} = [@storePaths];
}
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 43

[6.329]

[6.329]

getChannelData($c, 1);
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 56

[29.177]

[6.737]

getChannelData($c, 1);
replacement in src/lib/Hydra/Base/Controller/NixChannel.pm at line 63

[6.84]→[6.84:132](∅→∅)

my $pkg = $c->stash->{nixPkgs}->{$pkgName};

[6.84]

[6.132]

if (!$c->stash->{build}) {
$pkgName =~ /-(\d+)\.nixpkg$/ or notFound($c, "Bad package name.");
$c->stash->{build} = $c->stash->{channelBuilds}->find({ id => $1 })
|| notFound($c, "No such package in this channel.");
}
replacement in src/lib/Hydra/Base/Controller/NixChannel.pm at line 69

[6.133]→[6.133:215](∅→∅),[6.215]→[6.0:40](∅→∅)

notFound($c, "Unknown Nix package `$pkgName'.")
unless defined $pkg;

$c->stash->{build} = $pkg->{build};

[6.133]

[6.0]

if (!isValidPath($c->stash->{build}->outpath)) {
$c->response->status(410); # "Gone"
error($c, "Build " . $c->stash->{build}->id . " is no longer available.");
}
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 85

[29.265]

[6.41]

getChannelData($c, 1);
edit in src/lib/Hydra/Base/Controller/NixChannel.pm at line 105

[6.228]

[6.228]

# Optimistically assume that none of the packages have been
# garbage-collected. That should be true for the "latest"
# channel.
getChannelData($c, 0);
replacement in src/lib/Hydra/Controller/Build.pm at line 373

[6.1791]→[2.0:49](∅→∅),[2.49]→[6.0:5](∅→∅),[6.49]→[6.0:5](∅→∅),[6.1881]→[6.0:5](∅→∅),[6.5]→[6.404:551](∅→∅)

$c->stash->{storePaths} = [$build->outpath];

my $pkgName = $build->nixname . "-" . $build->system;
$c->stash->{nixPkgs} = {"${pkgName}.nixpkg" => {build => $build, name => $pkgName}};

[6.1791]

[6.56]

$c->stash->{channelBuilds} = $c->model('DB::Builds')->search({id => $build->id});
edit in src/lib/Hydra/Helper/CatalystUtils.pm at line 100

[6.735]→[6.10565:10566](∅→∅),[6.10565]→[6.10565:10566](∅→∅)
edit in src/lib/Hydra/Helper/CatalystUtils.pm at line 101

[6.625]→[6.625:673](∅→∅)

sub getChannelData {
my ($c, $builds) = @_;
edit in src/lib/Hydra/Helper/CatalystUtils.pm at line 102

[6.1999]→[5.801:1028](∅→∅),[6.117]→[6.673:678](∅→∅),[5.1028]→[6.673:678](∅→∅),[6.2158]→[6.673:678](∅→∅),[6.5186]→[6.673:678](∅→∅),[6.673]→[6.673:678](∅→∅),[6.749]→[6.749:774](∅→∅),[6.774]→[6.2159:2194](∅→∅),[6.2194]→[6.161:255](∅→∅),[6.255]→[6.94:291](∅→∅),[6.2493]→[6.94:291](∅→∅),[6.94]→[6.94:291](∅→∅),[6.291]→[3.0:40](∅→∅),[3.40]→[6.338:348](∅→∅),[6.338]→[6.338:348](∅→∅),[6.348]→[6.552:724](∅→∅),[6.1034]→[6.552:724](∅→∅),[6.724]→[3.41:378](∅→∅),[3.378]→[6.1178:1231](∅→∅),[6.724]→[6.1178:1231](∅→∅),[6.1178]→[6.1178:1231](∅→∅),[6.1231]→[6.3706:3710](∅→∅),[6.11101]→[6.3706:3710](∅→∅),[6.3706]→[6.3706:3710](∅→∅)

my @builds2 = $builds
->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')")
->search({}, { columns => [@buildListColumns, 'drvpath', 'outpath', 'description', 'homepage'] });

my @storePaths = ();
foreach my $build (@builds2) {
next unless isValidPath($build->outpath);
if (isValidPath($build->drvpath)) {
# Adding `drvpath' implies adding `outpath' because of the
# `--include-outputs' flag passed to `nix-store'.
push @storePaths, $build->drvpath;
} else {
push @storePaths, $build->outpath;
}
my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id;
$c->stash->{nixPkgs}->{"${pkgName}.nixpkg"} = {build => $build, name => $pkgName};
# Put the system type in the manifest (for top-level paths) as
# a hint to the binary patch generator. (It shouldn't try to
# generate patches between builds for different systems.) It
# would be nice if Nix stored this info for every path but it
# doesn't.
$c->stash->{systemForPath}->{$build->outpath} = $build->system;
};

$c->stash->{storePaths} = [@storePaths];
}