grahamc/hydra - Change HE3GX5IPS6XA7KLNP5WZY2YU27WQS663WSKAL5FHWBSJ5GMM633AC

Optimize fetch-git.

Created by Nicolas Pierron on April 14, 2012

HE3GX5IPS6XA7KLNP5WZY2YU27WQS663WSKAL5FHWBSJ5GMM633AC

Dependencies

In channels

upstream

main

Change contents

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 318 [12.2663]

B:BD[11.432] → [2.0:100]

    (my $uri, my $branch) = split ' ', $value;
    $branch = defined $branch ? $branch : "master";

[11.432]

[2.100]

    (my $uri, my $branch, my $deepClone) = split ' ', $value;
    $branch = defined $branch ? $branch : "master";

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 328 [12.2663]
∅:D[13.180] → [14.92:120]
B:BD[14.92] → [14.92:120]
```
    my $stdout; my $stderr;
```
[13.180]
[13.181]
```
    my $stdout = ""; my $stderr = ""; my $res;
```

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 330 [12.2663]

B:BD[13.208] → [13.208:271]

        (my $res, $stdout, $stderr) = captureStdoutStderr(600,

[13.208]

[15.0]

        # Clone everything and fetch the branch.
        # TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
        ($res, $stdout, $stderr) = captureStdoutStderr(600,

Deletion in src/lib/Hydra/Helper/AddBuilds.pm at line 337 [12.2663]
B:BD[13.397] → [13.397:424]
```
    # git pull + check rev
```

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 338 [12.2663]

∅:D[16.227] → [17.116:175]

∅:D[13.456] → [17.116:175]

B:BD[14.120] → [17.116:175]

B:BD[17.175] → [10.55:90]

∅:D[18.86] → [13.483:563]

∅:D[10.90] → [13.483:563]

∅:D[15.97] → [13.483:563]

B:BD[13.483] → [13.483:563]

    (my $res, $stdout, $stderr) = captureStdoutStderr(600,
        ("git", "pull", "--all"));
    die "Error pulling latest change git repo at `$uri':\n$stderr" unless $res;

[16.227]

[13.563]


    if (defined $deepClone) {
        # This fetch every branches from the remote repository and create a
        # local branch for each heads of the remote repository.  This is
        # necessary to provide a working git-describe.
        ($res, $stdout, $stderr) = captureStdoutStderr(600,
            ("git", "pull", "--ff-only", "-fu", "--all", "origin"));
        die "Error pulling latest change from git repo at `$uri':\n$stderr" unless $res;
    } else {
        # This command force the update of the local branch to be in the same as
        # the remote branch for whatever the repository state is.  This command mirror
        # only one branch of the remote repository.
        ($res, $stdout, $stderr) = captureStdoutStderr(600,
            ("git", "fetch", "-fu", "origin", "+$branch:$branch"));
        die "Error fetching latest change from git repo at `$uri':\n$stderr" unless $res;
    }

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 355 [12.2663]

B:BD[13.564] → [13.564:624]

B:BD[13.624] → [19.0:151]

    (my $res1, $stdout, $stderr) = captureStdoutStderr(600,
        ("git", "ls-remote", $clonePath, $branch));
    
    die "Cannot get head revision of Git branch '$branch' at `$uri':\n$stderr" unless $res1 ;

[13.564]

[11.515]

    ($res, $stdout, $stderr) = captureStdoutStderr(600,
        ("git", "rev-parse", "$branch"));
    die "Error getting revision number of Git branch '$branch' at `$uri':\n$stderr" unless $res;

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 359 [12.2663]

B:BD[11.516] → [10.91:309]

∅:D[10.309] → [19.152:240]

B:BD[11.516] → [19.152:240]

    # Take the first commit ID returned by `ls-remote'.  The
    # assumption is that `ls-remote' returned both `refs/heads/BRANCH'
    # and `refs/remotes/origin/BRANCH', and that both point at the
    # same commit.
    my ($first) = split /\n/, $stdout;
    (my $revision, my $ref) = split ' ', $first;

[11.516]

[14.359]

    my ($revision) = split /\n/, $stdout;

Insertion in src/lib/Hydra/Helper/AddBuilds.pm at line 361 [12.2663]

[14.405]

    die "Error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout" unless $res;

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 363 [12.2663]

B:BD[14.406] → [20.0:25]

B:BD[20.25] → [8.1163:1403]

    if (-f ".topdeps") {
        # This is a TopGit branch.  Fetch all the topic branches so
        # that builders can run "tg patch" and similar.
        (my $res, $stdout, $stderr) = captureStdoutStderr(600,
          ("tg", "remote", "--populate", "origin"));

[14.406]

[20.237]

    my $ref = "refs/heads/$branch";
    # If deepClone is defined, then we look at the content of the repository
    # to determine if this is a top-git branch.
    if (defined $deepClone) {
        # Checkout the branch to look at its content.
        ($res, $stdout, $stderr) = captureStdoutStderr(600,
            ("git", "checkout", "$branch"));
        die "Error checking out Git branch '$branch' at `$uri':\n$stderr" unless $res;

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 374 [12.2663]

B:BD[20.238] → [8.1404:1496]

        print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;

[20.238]

[20.323]

        if (-f ".topdeps") {
            # This is a TopGit branch.  Fetch all the topic branches so
            # that builders can run "tg patch" and similar.
            ($res, $stdout, $stderr) = captureStdoutStderr(600,
                ("tg", "remote", "--populate", "origin"));
            print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;
        }

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 384 [12.2663]

∅:D[20.330] → [14.406:532]

B:BD[14.406] → [14.406:532]

    # Some simple caching: don't check a uri/branch more than once every hour, but prefer exact match on uri/branch/revision.

[20.330]

[14.532]

    # Some simple caching: don't check a uri/branch/revision more than once.
    # TODO: Fix case where the branch is reset to a previous commit.

Deletion in src/lib/Hydra/Helper/AddBuilds.pm at line 396 [12.2663]
B:BD[11.960] → [21.172:177]
```
    
```

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 397 [12.2663]

B:BD[22.6670] → [3.0:59]

        print STDERR "checking out Git input from $uri\n";

[22.6670]

[22.6727]

        print STDERR "checking out Git branch $branch from $uri\n";

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 400 [12.2663]

∅:D[22.6803] → [21.352:357]

B:BD[21.352] → [21.352:357]

B:BD[21.357] → [22.6804:7115]

∅:D[22.7169] → [21.704:709]

B:BD[21.704] → [21.704:709]

B:BD[21.709] → [22.7170:7419]

    
        # Checked out code often wants to be able to run `git
        # describe', e.g., code that uses Gnulib's `git-version-gen'
        # script.  Thus, we leave `.git' in there.  Same for
        # Subversion (e.g., libgcrypt's build system uses that.)
        $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
    
        # Ask for a "deep clone" to allow "git describe" and similar
        # tools to work.  See
        # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
        # for a discussion.
        $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";

[22.6803]

[9.0]

        $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
        $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
        if (defined $deepClone) {
            # Checked out code often wants to be able to run `git
            # describe', e.g., code that uses Gnulib's `git-version-gen'
            # script.  Thus, we leave `.git' in there.  Same for
            # Subversion (e.g., libgcrypt's build system uses that.)
            $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
            # Ask for a "deep clone" to allow "git describe" and similar
            # tools to work.  See
            # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
            # for a discussion.
            $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
        }

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 417 [12.2663]

∅:D[9.1] → [17.236:299]

B:BD[21.948] → [17.236:299]

        (my $res, $stdout, $stderr) = captureStdoutStderr(600,

[9.1]

[5.0]

        ($res, $stdout, $stderr) = captureStdoutStderr(600,

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 537 [12.2663]
B:BD[23.402] → [23.402:430]
```
    my $stdout; my $stderr;
```
[23.402]
[24.1290]
```
    my $stdout = ""; my $stderr = "";
```
Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 709 [12.2663]
B:BD[8.1567] → [8.1567:1605]
```
        return (undef, undef, undef);
```
[8.1567]
[25.116]
```
        return (undef, "", "timeout\n");
```
Replacement in src/script/hydra-build at line 374 [27.2912]
B:BD[26.4205] → [4.0:11]
```
        } 
```
[26.4205]
[27.3728]
```
        }
```

Replacement in src/script/nix-prefetch-git at line 66 [28.1944]

B:BD[28.3302] → [28.3302:3401]

      --deepClone     Clone submodules recursively.
      --no-deepClone  Do not clone submodules.

[28.3302]

[28.3401]

      --deepClone     Clone history until a tag is found as parent.
      --no-deepClone  Clone the minimum history.

Replacement in src/script/nix-prefetch-git at line 120 [28.1944]

B:BD[28.4665] → [28.4665:4675]

	return 1

[28.4665]

[28.4675]


        # To make git describe works, we need to fetch all tags.
        if git fetch -t ${builder:+--progress} --depth 1 origin; then
	    return 1
        else
            # There is no tag, don't try to recover git-describe mechanism.
            deepClone=false
        fi

Replacement in src/script/nix-prefetch-git at line 219 [28.1944]
B:BD[28.6957] → [28.6957:6989]
```
	echo "removing \`.git'..." >&2
```
[28.6957]
[28.6989]
```
	test -n "$QUIET" || echo "removing \`.git'..." >&2
```

Replacement in src/script/nix-prefetch-git at line 258 [28.1944]

B:BD[28.8025] → [28.8025:8088]

      if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi

[28.8025]

[28.8088]

      test -n "$QUIET" || echo "hash is $hash" >&2;

Replacement in src/script/nix-prefetch-git at line 269 [28.1944]

B:BD[28.8352] → [28.8352:8416]

  if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi

[28.8352]

[28.8416]

  test -n "$QUIET" || echo "path is $finalPath" >&2

Replacement in tests/Setup.pm at line 11 [29.518]

B:BD[29.587] → [30.248:407]

our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild);

[29.587]

[29.618]

our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);

Insertion in tests/Setup.pm at line 75 [29.518]

[7.440]

[30.822]

}
sub updateRepository {
  my ($scm, $update, $repo) = @_;
  my ($res, $stdout, $stderr) = captureStdoutStderr(60, ($update, $repo));
  die "Unexpected update error with $scm: $stderr\n" unless $res;
  print STDOUT "Update $scm repository: $stdout" if $stdout ne "";
  return $stdout ne "";

Replacement in tests/evaluation-tests.pl at line 10 [29.804]
B:BD[29.969] → [6.165:195]
```
use Test::Simple tests => 28;
```
[29.969]
[29.998]
```
use Test::Simple tests => 48;
```

Replacement in tests/evaluation-tests.pl at line 60 [29.804]

B:BD[31.253] → [32.2401:2625]

∅:D[31.357] → [33.2054:2056]

∅:D[32.2625] → [33.2054:2056]

B:BD[33.2054] → [33.2054:2056]

  ok(evalSucceeds($jobset),                 "Evaluating jobs/$scm-input.nix should exit with return code 0.");
  ok(nrQueuedBuildsForJobset($jobset) == 1, "Evaluating jobs/$scm-input.nix should result in 1 build in queue");
}

[31.253]

[34.0]

  my $c = 1;
  my $q = 1;
  do {
    # Verify that it can be fetched and queued.
    ok(evalSucceeds($jobset),                  "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
    ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in 1 build in queue"); $c++;

Insertion in tests/evaluation-tests.pl at line 67 [29.804]

[34.1]

    # Verify that it is deterministic and not queued again.
    ok(evalSucceeds($jobset),                  "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
    ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in $q build in queue"); $c++;
    $q++;
  } while(updateRepository($scm, getcwd . "/jobs/$scm-update.sh", getcwd . "/$scm-repo/"));
}

File addition: bzr-checkout-update.sh (---x------)
[29.1729]
```
#! /bin/sh
```
File addition: bzr-update.sh (---x------)
[29.1729]
```
#! /bin/sh
```

File addition: git-update.sh (---x------)

[29.1729]

#! /bin/sh
cd "$1"
STATE_FILE=.state
if test -e $STATE_FILE; then
    state=$(cat $STATE_FILE)
else
    state=0;
fi
case $state in
    (0)
    echo "Add new file."
    touch git-file-2
    git add git-file-2 >&2
    git commit -m "add git file 2" git-file-2 >&2
    ;;
    (1)
    echo "Rewrite commit."
    echo 1 > git-file-2
    git add git-file-2 >&2
    git commit --amend -m "add git file 2" git-file-2 >&2
    ;;
esac
echo $(($state + 1)) > $STATE_FILE

File addition: hg-update.sh (---x------)
[29.1729]
```
#! /bin/sh
```
File addition: svn-checkout-update.sh (---x------)
[29.1729]
```
#! /bin/sh
```
File addition: svn-update.sh (---x------)
[29.1729]
```
#! /bin/sh
```

Optimize fetch-git.

Dependencies

In channels

Change contents

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 318 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 328 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 330 [12.2663]

Deletion in src/lib/Hydra/Helper/AddBuilds.pm at line 337 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 338 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 355 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 359 [12.2663]

Insertion in src/lib/Hydra/Helper/AddBuilds.pm at line 361 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 363 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 374 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 384 [12.2663]

Deletion in src/lib/Hydra/Helper/AddBuilds.pm at line 396 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 397 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 400 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 417 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 537 [12.2663]

Replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 709 [12.2663]

Replacement in src/script/hydra-build at line 374 [27.2912]

Replacement in src/script/nix-prefetch-git at line 66 [28.1944]

Replacement in src/script/nix-prefetch-git at line 120 [28.1944]

Replacement in src/script/nix-prefetch-git at line 219 [28.1944]

Replacement in src/script/nix-prefetch-git at line 258 [28.1944]

Replacement in src/script/nix-prefetch-git at line 269 [28.1944]

Replacement in tests/Setup.pm at line 11 [29.518]

Insertion in tests/Setup.pm at line 75 [29.518]

Replacement in tests/evaluation-tests.pl at line 10 [29.804]

Replacement in tests/evaluation-tests.pl at line 60 [29.804]

Insertion in tests/evaluation-tests.pl at line 67 [29.804]

File addition: bzr-checkout-update.sh (---x------)

File addition: bzr-update.sh (---x------)

File addition: git-update.sh (---x------)

File addition: hg-update.sh (---x------)

File addition: svn-checkout-update.sh (---x------)

File addition: svn-update.sh (---x------)