* Caching of "path" inputs, and fake a revision number for those.

[?]

Nov 25, 2008, 2:59 PM

6BLUKEQ2M5RGWMPXPYIFIEVEUBV4PYAZ75S2WSBIATMRGYFMQZHQC

Dependencies

[2] H7CNGK4O * Log evaluation errors etc. in the DB.
[3] 7YBYT2LQ
[4] FDE3BJAP * Refactoring.
[5] WYN733ST * Store build duration, handle cached builds.
[6] VCOSLZRP
[7] ZEHSSVFG
[8] TLZ2SPBR
[9] PHX2HIVG * Store info about the build inputs in the build record.
[10] M552HLIA * Support variant builds.
[11] BVOPAMLS
[12] X27GNHDV * Basic job info in the database.
[13] L2E6EVE2 * Merged the Build and Job tables.
[14] CLJQCY2X * Store info about all the build actions and allow them to be
[15] B72GLND4
[16] DVNWJXWW * Generic declaration of build products.
[17] KOTB7BKV
[18] HJLYC753 * Adding input value alternatives.
[19] IK53RV4V
[20] SHBLLAVH * More global substitution.
[21] J5UVLXOK * Start of a basic Catalyst web interface.
[22] YTZOC7C5 * Editing of jobset inputs.
[23] BD3GRK4B * Get rid of "positive failures" and separate log phases.
[24] TQKGQ5R3
[25] GWCV3TQV * BuildInputs table: link to dependencies, include store paths.
[26] 67P45PY4
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

replacement in src/Hydra/lib/Hydra/Schema/Buildinputs.pm at line 39

[3.782]→[2.253:395](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw

[3.782]

[3.924]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Nv28F7I+7azffAhYPWYTvA
replacement in src/Hydra/lib/Hydra/Schema/Buildproducts.pm at line 36

[3.4922]→[2.396:538](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA

[3.4922]

[3.5064]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fj95oLQNEptO9LgSfclVGQ
replacement in src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm at line 30

[3.2382]→[2.539:681](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA

[3.2382]

[3.2524]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CvVVHVHL2UEZ5XMkgf01OA
replacement in src/Hydra/lib/Hydra/Schema/Builds.pm at line 73

[3.6194]→[2.682:824](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ

[3.6194]

[3.3377]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ty4vMjEqpl6yM81PrOtjqA
replacement in src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm at line 26

[3.4473]→[2.825:967](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA

[3.4473]

[3.4615]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:81cloGb6HT/PTPgzB8v5Tg
replacement in src/Hydra/lib/Hydra/Schema/Buildsteps.pm at line 38

[3.2816]→[2.968:1110](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ

[3.2816]

[3.2958]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cNgqL+ZLxQirdHUa/i6mww
replacement in src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm at line 34

[3.1831]→[2.1111:1253](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw

[3.1831]

[3.1973]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zZEUHrrOIOIg1JH0Pgi5xg
replacement in src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm at line 46

[3.1681]→[2.1254:1396](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw

[3.1681]

[3.1823]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uzlil1U0VlHnTe1tvaRO4A
replacement in src/Hydra/lib/Hydra/Schema/Jobsets.pm at line 51

[3.2992]→[2.1551:1693](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg

[3.2992]

[3.3134]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ednIpbErLs3gjqQd9w4KsA
replacement in src/Hydra/lib/Hydra/Schema/Projects.pm at line 33

[3.3685]→[2.1694:1836](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw

[3.3685]

[3.3827]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OGLaq1Y/rLPlz+whXTZnzw
replacement in src/Hydra/lib/Hydra/Schema.pm at line 11

[3.6498]→[2.1837:1979](∅→∅)

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ

[3.6498]

[3.6640]

# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 15:16:17
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:scYtOwO4xnXnKWlNGWyjcQ
edit in src/Hydra/programs/Scheduler.pl at line 7

[2.1994]

[28.1392]

use POSIX qw(strftime);
replacement in src/Hydra/programs/Scheduler.pl at line 48

[3.2738]→[2.2175:2246](∅→∅),[2.2246]→[3.6820:6956](∅→∅),[3.2738]→[3.6820:6956](∅→∅),[3.5346]→[3.6820:6956](∅→∅),[3.6820]→[3.6820:6956](∅→∅),[3.50]→[3.2187:2196](∅→∅)

print "copying input ", $input->name, " from $uri\n";

my $storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;

[3.2738]

[3.2196]

my $timestamp = time;
my $sha256;
my $storePath;

# Some simple caching: don't check a path more than once every N seconds.
(my $cachedInput) = $db->resultset('Cachedpathinputs')->search(
{srcpath => $uri, lastseen => {">", $timestamp - 60}},
{rows => 1, order_by => "lastseen DESC"});

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
print "CACHED $uri $cachedInput ", $cachedInput->timestamp, " ", $cachedInput->timestamp, "\n";
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
} else {

print "copying input ", $input->name, " from $uri\n";
$storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;

$sha256 = getStorePathHash $storePath;

($cachedInput) = $db->resultset('Cachedpathinputs')->search(
{srcpath => $uri, sha256hash => $sha256});

# Path inputs don't have a natural notion of a "revision",
# so we simulate it by using the timestamp that we first
# saw this path have this SHA-256 hash. So if the
# contents of the path changes, we get a new "revision",
# but if it doesn't change (or changes back), we don't get
# a new "revision".
if (!defined $cachedInput) {
$db->txn_do(sub {
$db->resultset('Cachedpathinputs')->create(
{ srcpath => $uri
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});
});
} else {
$timestamp = $cachedInput->timestamp;
$db->txn_do(sub {
$cachedInput->lastseen(time);
$cachedInput->update;
});
}
}
replacement in src/Hydra/programs/Scheduler.pl at line 104

[3.2324]→[3.2324:2380](∅→∅)

, sha256hash => getStorePathHash $storePath

[3.2324]

[3.2380]

, sha256hash => $sha256
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
replacement in src/Hydra/programs/Scheduler.pl at line 183

[3.10742]→[3.10742:10844](∅→∅)

#, revision => $input->{orig}->revision
#, tag => $input->{orig}->tag

[3.10742]

[3.10844]

, revision => $input->{revision}
replacement in src/Hydra/programs/Scheduler.pl at line 231

[2.3477]→[3.1838:1902](∅→∅),[3.1838]→[3.1838:1902](∅→∅)

fetchInput($input, $alt, $inputInfo); # !!! caching

[2.3477]

[2.3478]

fetchInput($input, $alt, $inputInfo);
replacement in src/Hydra/programs/Scheduler.pl at line 234

[3.1995]→[2.3520:3627](∅→∅)

push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}";

[3.1995]

[3.2128]

push @newArgs, "--arg", $argName,
"{path = " . $inputInfo->{$argName}->{storePath} . ";" .
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
replacement in src/Hydra/root/build.tt at line 132

[3.2245]→[3.1370:1442](∅→∅)

<tr><th>Name</th><th>Type</th><th>What</th><th>Store path</th></tr>

[3.2245]

[3.2297]

<tr><th>Name</th><th>Type</th><th>Value</th><th>Revision</th><th>Store path</th></tr>
edit in src/Hydra/root/build.tt at line 148

[3.1767]

[3.1767]

<td>[% IF input.revision %][% input.revision %][% END %]</td>
edit in src/hydra.sql at line 227

[3.11357]

[3.5642]

);

-- Cache for inputs of type "path" (used for testing Hydra), storing
-- the SHA-256 hash and store path for each source path. Also stores
-- the timestamp when we first saw the path have these contents, which
-- may be used to generate release names.
create table CachedPathInputs (
srcPath text not null,
timestamp integer not null, -- when we first saw this hash
lastSeen integer not null, -- when we last saw this hash
sha256hash text not null,
storePath text not null,
primary key (srcPath, sha256hash)