Optimize fetch-git.

[?]

Apr 15, 2012, 1:17 AM

HE3GX5IPS6XA7KLNP5WZY2YU27WQS663WSKAL5FHWBSJ5GMM633AC

Dependencies

[2] ENYUXIFP support git branches as input in stead of only master branch
[3] VXY434VY
[4] UVNQPK3T Hydra/56: handle failed builds with result only at build level, not buildsteps
[5] KW3M6NSR hydra: run nix-prefetch-git on local clone of git repo
[6] RBZRCTAL hydra: fixed and re-enabled evaluation tests
[7] V2ALOFRR Fix the test
[8] EDRUQ4UK Die TABs die
[9] 4FWDVNWA Pass additional attributes for Git inputs
[10] DAKY4N64 Allow use of a single Git clone for several branches.
[11] 5SMQ2PLK Fix tests
[12] HABC6L4G Revert "Use git fetch --all to update local clone in stead of git pull. Also, do not use --branch in initial clone."
[13] KQS7DSKJ * Clean up indentation.
[14] BMSQD2ZH Indentation
[15] G3IUM7VL hydra: add support for succeedOnFailure feature in stdenv
[16] WFJ3ZPRM add tests for scm inputs
[17] 3I6BNF4S hydra: do not use refs/heads while checking for latest commit in branch
[18] V2MLUZRP Random indentation fix
[19] 6ZSLRO7M fetchInputGit: Populate TopGit branches when on a topic branch.
[20] VJ4BZV5R Add branch to local clone path of git repositories for now.
[21] ARD6Z67T Do incremental SVN checkouts
[22] 2WUNXJGW Hydra/26: Go back to using "svn export" as default for svn, added svn-checkout type for jobset which need .svn dirs. export is much more efficient
[23] UO67NQMR Change `fetchInputGit' to do `git clone -b BRANCH' instead of `git clone'.
[24] WWUOQ7V4 * hydra: indentation and fixed duplicate key in cachedgitinputs bug
[25] CHQEG6WY Hydra/29: Added timeout to svn/git checkouts, job evaluation
[26] O25D52TA initial support for mercurial
[27] SBKX3YH2 cleanup
[28] FAIJDQKZ
[29] HX4QYOYA add first evaluations tests
[30] HQ4QHAOF make default branch default, remove quotes
[31] ZHBEOOUQ show evaluation errors in log of tests
[32] 7YBYT2LQ
[33] KX5L74EY add nix-prefetch- scripts for now, were externals in svn
[34] TM6XBAG2 git branches can share local clones
[35] BK24VA6Q * for git inputs, check latest revision of branch (defaults to master for now), if there is change, only use input if last checkout was > hour ago.
[36] C7CXMZ66 hydra: do not perform git clone every time. in stead work on local clone and pull
[37] M73H45NE add support for mercurial tags/revs
[38] PNPT3Y7R Use git fetch --all to update local clone in stead of git pull. Also, do not use --branch in initial clone.
[39] BD3GRK4B * Get rid of "positive failures" and separate log phases.
[40] W6J4PVRP remove debug statement
[41] 2WRTOU2Z Cleanup
[42] JTRG7RDQ add support for git as jobinput
[*] OOQ2D3KC * Refactoring: move fetchInput out of hydra_scheduler into a separate

Change contents

replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 318

[11.432]→[2.0:100](∅→∅)

(my $uri, my $branch) = split ' ', $value;
$branch = defined $branch ? $branch : "master";

[11.432]

[2.100]

(my $uri, my $branch, my $deepClone) = split ' ', $value;
$branch = defined $branch ? $branch : "master";
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 328

[11.180]→[11.92:120](∅→∅),[11.92]→[11.92:120](∅→∅)

my $stdout; my $stderr;

[11.180]

[11.181]

my $stdout = ""; my $stderr = ""; my $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 330

[11.208]→[11.208:271](∅→∅)

(my $res, $stdout, $stderr) = captureStdoutStderr(600,

[11.208]

[11.0]

# Clone everything and fetch the branch.
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
($res, $stdout, $stderr) = captureStdoutStderr(600,
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 337

[11.397]→[11.397:424](∅→∅)

# git pull + check rev
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 338

[11.227]→[11.116:175](∅→∅),[11.456]→[11.116:175](∅→∅),[11.120]→[11.116:175](∅→∅),[11.175]→[10.55:90](∅→∅),[11.86]→[11.483:563](∅→∅),[10.90]→[11.483:563](∅→∅),[11.97]→[11.483:563](∅→∅),[11.483]→[11.483:563](∅→∅)

(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "pull", "--all"));
die "Error pulling latest change git repo at `$uri':\n$stderr" unless $res;

[11.227]

[11.563]

if (defined $deepClone) {
# This fetch every branches from the remote repository and create a
# local branch for each heads of the remote repository. This is
# necessary to provide a working git-describe.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "pull", "--ff-only", "-fu", "--all", "origin"));
die "Error pulling latest change from git repo at `$uri':\n$stderr" unless $res;
} else {
# This command force the update of the local branch to be in the same as
# the remote branch for whatever the repository state is. This command mirror
# only one branch of the remote repository.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "fetch", "-fu", "origin", "+$branch:$branch"));
die "Error fetching latest change from git repo at `$uri':\n$stderr" unless $res;
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 355

[11.564]→[11.564:624](∅→∅),[11.624]→[11.0:151](∅→∅)

(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
("git", "ls-remote", $clonePath, $branch));

die "Cannot get head revision of Git branch '$branch' at `$uri':\n$stderr" unless $res1 ;

[11.564]

[11.515]

($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "rev-parse", "$branch"));
die "Error getting revision number of Git branch '$branch' at `$uri':\n$stderr" unless $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 359

[11.516]→[10.91:309](∅→∅),[10.309]→[11.152:240](∅→∅),[11.516]→[11.152:240](∅→∅)

# Take the first commit ID returned by `ls-remote'. The
# assumption is that `ls-remote' returned both `refs/heads/BRANCH'
# and `refs/remotes/origin/BRANCH', and that both point at the
# same commit.
my ($first) = split /\n/, $stdout;
(my $revision, my $ref) = split ' ', $first;

[11.516]

[11.359]

my ($revision) = split /\n/, $stdout;
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 361

[11.405]

[11.405]

die "Error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout" unless $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 363

[11.406]→[11.0:25](∅→∅),[11.25]→[8.1163:1403](∅→∅)

if (-f ".topdeps") {
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("tg", "remote", "--populate", "origin"));

[11.406]

[11.237]

my $ref = "refs/heads/$branch";

# If deepClone is defined, then we look at the content of the repository
# to determine if this is a top-git branch.
if (defined $deepClone) {

# Checkout the branch to look at its content.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "checkout", "$branch"));
die "Error checking out Git branch '$branch' at `$uri':\n$stderr" unless $res;
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 374

[11.238]→[8.1404:1496](∅→∅)

print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;

[11.238]

[11.323]

if (-f ".topdeps") {
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("tg", "remote", "--populate", "origin"));

print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 384

[11.330]→[11.406:532](∅→∅),[11.406]→[11.406:532](∅→∅)

# Some simple caching: don't check a uri/branch more than once every hour, but prefer exact match on uri/branch/revision.

[11.330]

[11.532]

# Some simple caching: don't check a uri/branch/revision more than once.
# TODO: Fix case where the branch is reset to a previous commit.
edit in src/lib/Hydra/Helper/AddBuilds.pm at line 396

[11.960]→[11.172:177](∅→∅)
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 397

[11.6670]→[3.0:59](∅→∅)

print STDERR "checking out Git input from $uri\n";

[11.6670]

[11.6727]

print STDERR "checking out Git branch $branch from $uri\n";
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 400

[11.6803]→[11.352:357](∅→∅),[11.352]→[11.352:357](∅→∅),[11.357]→[11.6804:7115](∅→∅),[11.7169]→[11.704:709](∅→∅),[11.704]→[11.704:709](∅→∅),[11.709]→[11.7170:7419](∅→∅)

# Checked out code often wants to be able to run `git
# describe', e.g., code that uses Gnulib's `git-version-gen'
# script. Thus, we leave `.git' in there. Same for
# Subversion (e.g., libgcrypt's build system uses that.)
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";

# Ask for a "deep clone" to allow "git describe" and similar
# tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion.
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";

[11.6803]

[9.0]

$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";

if (defined $deepClone) {
# Checked out code often wants to be able to run `git
# describe', e.g., code that uses Gnulib's `git-version-gen'
# script. Thus, we leave `.git' in there. Same for
# Subversion (e.g., libgcrypt's build system uses that.)
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";

# Ask for a "deep clone" to allow "git describe" and similar
# tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion.
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
}
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 417

[9.1]→[11.236:299](∅→∅),[11.948]→[11.236:299](∅→∅)

(my $res, $stdout, $stderr) = captureStdoutStderr(600,

[9.1]

[5.0]

($res, $stdout, $stderr) = captureStdoutStderr(600,
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 537

[11.402]→[11.402:430](∅→∅)

my $stdout; my $stderr;

[11.402]

[11.1290]

my $stdout = ""; my $stderr = "";
replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 709

[8.1567]→[8.1567:1605](∅→∅)

return (undef, undef, undef);

[8.1567]

[11.116]

return (undef, "", "timeout\n");
replacement in src/script/hydra-build at line 374

[11.4205]→[4.0:11](∅→∅)

}

[11.4205]

[11.3728]

}
replacement in src/script/nix-prefetch-git at line 66

[11.3302]→[11.3302:3401](∅→∅)

--deepClone Clone submodules recursively.
--no-deepClone Do not clone submodules.

[11.3302]

[11.3401]

--deepClone Clone history until a tag is found as parent.
--no-deepClone Clone the minimum history.
replacement in src/script/nix-prefetch-git at line 120

[11.4665]→[11.4665:4675](∅→∅)

return 1

[11.4665]

[11.4675]

# To make git describe works, we need to fetch all tags.
if git fetch -t ${builder:+--progress} --depth 1 origin; then
return 1
else
# There is no tag, don't try to recover git-describe mechanism.
deepClone=false
fi
replacement in src/script/nix-prefetch-git at line 219

[11.6957]→[11.6957:6989](∅→∅)

echo "removing \`.git'..." >&2

[11.6957]

[11.6989]

test -n "$QUIET" || echo "removing \`.git'..." >&2
replacement in src/script/nix-prefetch-git at line 258

[11.8025]→[11.8025:8088](∅→∅)

if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi

[11.8025]

[11.8088]

test -n "$QUIET" || echo "hash is $hash" >&2;
replacement in src/script/nix-prefetch-git at line 269

[11.8352]→[11.8352:8416](∅→∅)

if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi

[11.8352]

[11.8416]

test -n "$QUIET" || echo "path is $finalPath" >&2
replacement in tests/Setup.pm at line 11

[11.587]→[11.248:407](∅→∅)

our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild);

[11.587]

[11.618]

our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
edit in tests/Setup.pm at line 75

[7.440]

[11.822]

}

sub updateRepository {
my ($scm, $update, $repo) = @_;
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ($update, $repo));
die "Unexpected update error with $scm: $stderr\n" unless $res;
print STDOUT "Update $scm repository: $stdout" if $stdout ne "";
return $stdout ne "";
replacement in tests/evaluation-tests.pl at line 10

[11.969]→[6.165:195](∅→∅)

use Test::Simple tests => 28;

[11.969]

[11.998]

use Test::Simple tests => 48;
replacement in tests/evaluation-tests.pl at line 60

[11.253]→[11.2401:2625](∅→∅),[11.357]→[11.2054:2056](∅→∅),[11.2625]→[11.2054:2056](∅→∅),[11.2054]→[11.2054:2056](∅→∅)

ok(evalSucceeds($jobset), "Evaluating jobs/$scm-input.nix should exit with return code 0.");
ok(nrQueuedBuildsForJobset($jobset) == 1, "Evaluating jobs/$scm-input.nix should result in 1 build in queue");
}

[11.253]

[11.0]

my $c = 1;
my $q = 1;
do {
# Verify that it can be fetched and queued.
ok(evalSucceeds($jobset), "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in 1 build in queue"); $c++;
edit in tests/evaluation-tests.pl at line 67

[11.1]

# Verify that it is deterministic and not queued again.
ok(evalSucceeds($jobset), "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in $q build in queue"); $c++;

$q++;
} while(updateRepository($scm, getcwd . "/jobs/$scm-update.sh", getcwd . "/$scm-repo/"));
}
file addition: bzr-checkout-update.sh (---r------)

[11.1729]

#! /bin/sh
file addition: bzr-update.sh (---r------)

[11.1729]

#! /bin/sh
file addition: git-update.sh (---r------)

[11.1729]

#! /bin/sh

cd "$1"
STATE_FILE=.state
if test -e $STATE_FILE; then
state=$(cat $STATE_FILE)
else
state=0;
fi

case $state in
(0)
echo "Add new file."
touch git-file-2
git add git-file-2 >&2
git commit -m "add git file 2" git-file-2 >&2
;;
(1)
echo "Rewrite commit."
echo 1 > git-file-2
git add git-file-2 >&2
git commit --amend -m "add git file 2" git-file-2 >&2
;;
esac

echo $(($state + 1)) > $STATE_FILE
file addition: hg-update.sh (---r------)

[11.1729]

#! /bin/sh
file addition: svn-checkout-update.sh (---r------)

[11.1729]

#! /bin/sh
file addition: svn-update.sh (---r------)

[11.1729]

#! /bin/sh