Do incremental SVN checkouts

[?]

Dec 4, 2011, 9:01 PM

ARD6Z67TDHK2XMW47CSVOXXF327YGOMYK3EAXTSVVXNDAFQYNO5AC

Dependencies

[2] KX5L74EY add nix-prefetch- scripts for now, were externals in svn
[3] JZE7DC2F Whitespace
[4] TM6XBAG2 git branches can share local clones
[5] L4AI5YL6 Rename hydra_*.pl to hydra-*
[6] C7CXMZ66 hydra: do not perform git clone every time. in stead work on local clone and pull
[7] BK24VA6Q * for git inputs, check latest revision of branch (defaults to master for now), if there is change, only use input if last checkout was > hour ago.
[8] O25D52TA initial support for mercurial
[9] ENYUXIFP support git branches as input in stead of only master branch
[10] DBPIYHMA hydra: add nix-prefetch-* to tarball
[11] V2MLUZRP Random indentation fix
[12] FXW2UR7F initial bzr support
[13] FGI75W53 move nix-prefetch-* into tarball
[14] KQS7DSKJ * Clean up indentation.
[15] FV2M6MOT hydra: use autoconf/-make
[16] 3XTHEUMP * Implemented the clone feature.
[17] HQ4QHAOF make default branch default, remove quotes
[18] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate
[19] JTRG7RDQ add support for git as jobinput
[20] BMSQD2ZH Indentation
[21] D5QIOJGP * Move everything up one directory.
[22] 6MGFQDR2 deleted some old scripts
[23] CHQEG6WY Hydra/29: Added timeout to svn/git checkouts, job evaluation
[24] UO67NQMR Change `fetchInputGit' to do `git clone -b BRANCH' instead of `git clone'.
[25] Q6SOGMDQ Hydra/28: Rename "scheduler" to "evaluator"
[26] 2WUNXJGW Hydra/26: Go back to using "svn export" as default for svn, added svn-checkout type for jobset which need .svn dirs. export is much more efficient
[27] M73H45NE add support for mercurial tags/revs
[28] FKTXLQVZ hydra: nix-prefetch-bzr missing in tarball

Change contents

edit in src/lib/Hydra/Helper/AddBuilds.pm at line 14

[6.196]

[6.2759]

use File::Temp;
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 186

[6.7352]

[6.2521]

# Do we already have this revision in the store?
# !!! This needs to take $checkout into account! Otherwise "svn"
# and "svn-checkout" inputs can get mixed up.
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 196

[6.2818]→[6.7673:7686](∅→∅),[6.7673]→[6.7673:7686](∅→∅),[6.7686]→[6.2819:3046](∅→∅),[6.3046]→[6.186:257](∅→∅)

# Then download this revision into the store.
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_SVN_LEAVE_DOT_SVN"} = "$checkout";

[6.2818]

[6.0]

# No, do a checkout. The working copy is reused between
# invocations to speed things up.
mkpath(scmPath . "/svn");
my $wcPath = scmPath . "/svn" . sha256_hex($uri) . "/svn-checkout";

print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 205

[6.63]→[6.63:115](∅→∅),[6.115]→[6.3155:3238](∅→∅),[6.3155]→[6.3155:3238](∅→∅)

("nix-prefetch-svn", $uri, $revision));
die "Cannot check out Subversion repository `$uri':\n$stderr" unless $res;

[6.63]

[6.8133]

("svn", "checkout", $uri, "-r", $revision, $wcPath));

if ($checkout) {
$storePath = addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 217

[6.8134]→[6.3239:3291](∅→∅)

($sha256, $storePath) = split ' ', $stdout;

[6.8134]

[6.8190]

$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 220

[6.3318]→[6.3318:3380](∅→∅)

$db->resultset('CachedSubversionInputs')->create(

[6.3318]

[6.3380]

$db->resultset('CachedSubversionInputs')->update_or_create(
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 328

[6.1]→[6.84:103](∅→∅)

my $clonePath;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 329

[6.124]→[4.59:110](∅→∅)

$clonePath = scmPath . "/" . sha256_hex($uri);

[6.124]

[6.179]

my $clonePath = scmPath . "/" . sha256_hex($uri);
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 427

[6.275]→[6.275:294](∅→∅)

my $clonePath;

[6.275]

[6.294]
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 429

[6.315]→[6.315:366](∅→∅)

$clonePath = scmPath . "/" . sha256_hex($uri);

[6.315]

[6.366]

my $clonePath = scmPath . "/" . sha256_hex($uri);
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 500

[6.430]→[6.430:449](∅→∅)

my $clonePath;

[6.430]

[6.449]
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 502

[6.470]→[6.470:521](∅→∅)

$clonePath = scmPath . "/" . sha256_hex($uri);

[6.470]

[6.521]

my $clonePath = scmPath . "/" . sha256_hex($uri);
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 643

[6.2323]→[6.352:385](∅→∅)

(my $timeout, my @cmd) = @_;

[6.2323]

[6.385]

my ($timeout, @cmd) = @_;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 665

[3.22]→[6.2469:2474](∅→∅),[6.837]→[6.2469:2474](∅→∅),[6.2469]→[6.2469:2474](∅→∅)

[3.22]

[6.2474]
file deletion: nix-prefetch-svn (---r------)

[6.2543]→[2.12079:12119](∅→∅),[2.12119]→[2.10237:10237](∅→∅)

#! /bin/sh -e

url=$1
rev=$2
expHash=$3

hashType=$NIX_HASH_ALGO
if test -z "$hashType"; then
hashType=sha256
fi
if test -z "$hashFormat"; then
hashFormat=--base32
fi

if test -z "$url"; then
echo "syntax: nix-prefetch-svn URL [REVISION [EXPECTED-HASH]]" >&2
exit 1
fi

test -n "$rev" || rev="HEAD"

repoName=$(echo $url | sed '
s,.*/$[^/]\+$/trunk/*$,\1,;t
s,.*/$[^/]\+$/branches/$[^/]\+$/*$,\1-\2,;t
s,.*/$[^/]\+$/tags/$[^/]\+$/*$,\1-\2,;t
s,.*/$[^/]\+$/*$,\1,;t
')
dstFile=$repoName-r$rev

# If the hash was given, a file with that hash may already be in the
# store.
if test -n "$expHash"; then
finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" $dstFile)
if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
finalPath=
fi
hash=$expHash
fi

# If we don't know the hash or a path with that hash doesn't exist,
# download the file and add it to the store.
if test -z "$finalPath"; then
tmpPath=/tmp/svn-checkout-tmp-$$
tmpFile=$tmpPath/$dstFile
mkdir $tmpPath

trap "rm -rf $tmpPath" EXIT

# Perform the checkout.
if test "$NIX_PREFETCH_SVN_LEAVE_DOT_SVN" != 1
then
command="export"
else
command="checkout"
fi

echo p | svn "$command" --quiet -r "$rev" "$url" "$tmpFile" >&2

# Compute the hash.
hash=$(nix-hash --type $hashType $hashFormat $tmpFile)
if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi

# Add the downloaded file to the Nix store.
finalPath=$(nix-store --add-fixed --recursive "$hashType" $tmpFile)

if test -n "$expHash" -a "$expHash" != "$hash"; then
echo "hash mismatch for URL \`$url'"
exit 1
fi
fi

if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi

echo $hash

if test -n "$PRINT_PATH"; then
echo $finalPath
fi
edit in src/script/Makefile.am at line 3

[5.18]→[6.36:78](∅→∅),[6.36]→[6.36:78](∅→∅),[6.78]→[6.0:21](∅→∅),[6.21]→[5.19:39](∅→∅)

nix-prefetch-svn \
nix-prefetch-git \
nix-prefetch-bzr \
nix-prefetch-hg \
edit in src/script/Makefile.am at line 12

[5.176]→[6.249:270](∅→∅),[6.2198832]→[6.249:270](∅→∅)

nix-prefetch-svn \