hydra: do not perform git clone every time. in stead work on local clone and pull

[?]

Jul 28, 2010, 12:48 PM

C7CXMZ66BZTI5ABDJNPL2JKSHSKDQ645GXUKN7OTQBPEQJ452KQAC

Dependencies

[2] VM3TQZ65 * hydra: make manual.pdf
[3] CHQEG6WY Hydra/29: Added timeout to svn/git checkouts, job evaluation
[4] O25D52TA initial support for mercurial
[5] KQS7DSKJ * Clean up indentation.
[6] 7C6HSXUQ * Start of a regression test for Hydra.
[7] 5DZPBNDE * hydra: on git ls-remote add refs/heads/ prefix to branchname to avoid other branches called master (or $branchname)
[8] BK24VA6Q * for git inputs, check latest revision of branch (defaults to master for now), if there is change, only use input if last checkout was > hour ago.
[9] ENYUXIFP support git branches as input in stead of only master branch
[10] WWUOQ7V4 * hydra: indentation and fixed duplicate key in cachedgitinputs bug
[11] G7XYM6MU use local clone to limit bandwidth usage
[12] YQWH4POV * Simplify.
[13] T4LLYESZ * Nix expression for building Hydra.
[14] JTRG7RDQ add support for git as jobinput
[15] ZVTSOVHN * Support Subversion checkouts.
[16] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
[*] N22GPKYT * Put info about logs / build products in the DB.

Change contents

replacement in release.nix at line 1

[5.1]→[5.0:23](∅→∅)

{ nixpkgs ? ../nixpkgs

[5.1]

[2.0]

{ nixpkgs ? /etc/nixos/nixpkgs
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 15

[5.2816]

[5.2816]

sub scmPath {
return getHydraPath . "/scm" ;
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 281

[5.30]→[5.30:92](∅→∅)

# First figure out the last-modified revision of the URI.

[5.1]

[5.92]

my $clonePath;
mkpath(scmPath);
$clonePath = scmPath . "/" . sha256_hex($uri);
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 286

[5.120]

[3.116]

if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "clone", $uri, $clonePath));
die "Error cloning git repo at `$uri':\n$stderr" unless $res;
}

# git pull + check rev
chdir $clonePath or die $!;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 295

[3.175]→[3.175:235](∅→∅),[5.58]→[5.216:308](∅→∅),[3.235]→[5.216:308](∅→∅),[5.6184]→[5.216:308](∅→∅),[5.216]→[5.216:308](∅→∅)

("git", "ls-remote", $uri, "refs/heads/".$branch));
die "Cannot get head revision of Git branch '$branch' at `$uri':\n$stderr" unless $res;

[3.175]

[5.515]

("git", "pull"));
die "Error pulling latest change git repo at `$uri':\n$stderr" unless $res;

(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
("git", "ls-remote", $clonePath, "refs/heads/".$branch));
die "Cannot get head revision of Git branch '$branch' at `$uri':\n$stderr" unless $res1;
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 310

[5.6272]→[5.691:726](∅→∅),[5.691]→[5.691:726](∅→∅),[5.726]→[5.6273:6480](∅→∅),[5.6480]→[5.912:918](∅→∅),[5.912]→[5.912:918](∅→∅)

if (! defined $cachedInput ) {
($cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, branch => $branch, lastseen => {">", $timestamp - 3600}},
{rows => 1, order_by => "lastseen DESC"});
}
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 314

[5.6525]→[5.6525:6571](∅→∅)

$timestamp = $cachedInput->timestamp;
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 339

[5.7678]→[5.7678:7816](∅→∅)

($cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, branch => $branch, sha256hash => $sha256});
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 340

[5.1335]→[5.7817:8276](∅→∅)

if (!defined $cachedInput) {
txn_do($db, sub {
$db->resultset('CachedGitInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});

[5.1335]

[5.8276]

txn_do($db, sub {
$db->resultset('CachedGitInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 348

[5.8296]→[5.8296:8393](∅→∅),[5.8393]→[5.1933:1991](∅→∅),[5.1933]→[5.1933:1991](∅→∅)

} else {
$timestamp = $cachedInput->timestamp;
txn_do($db, sub {
$cachedInput->update({lastseen => time});
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 349

[5.8410]→[5.8410:8420](∅→∅)

}
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 360

[5.2118]→[4.197:248](∅→∅),[4.248]→[5.2120:2121](∅→∅),[5.2120]→[5.2120:2121](∅→∅)

sub scmPath {
return getHydraPath . "/scm" ;
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 387

[4.1067]→[4.1067:1142](∅→∅)

die "Error getting head of $branch from `$uri':\n$stderr" unless $res;

[4.1067]

[4.1142]

die "Error getting head of $branch from `$uri':\n$stderr" unless $res1;
edit in src/sql/hydra.sql at line 314

[5.1776]→[5.6054:6187](∅→∅),[5.6054]→[5.6054:6187](∅→∅)

timestamp integer not null, -- when we first saw this hash
lastSeen integer not null, -- when we last saw this hash