* Clean up indentation.
[?]
Mar 5, 2010, 4:16 PM
KQS7DSKJHTEHALCPTXMWRX2IBOVC5BWU7RWJ4GP5A2JJSBR7HPKACDependencies
- [2]
WWUOQ7V4* hydra: indentation and fixed duplicate key in cachedgitinputs bug - [3]
RFE6T5LG* Store jobset evaluations in the database explicitly. This includes - [4]
OOQ2D3KC* Refactoring: move fetchInput out of hydra_scheduler into a separate - [5]
YFPZ46YK* hydra: added variant of build input type, 'build output (same system)' to allow better continous integration in one jobset for multiple system. it makes sure that the system of the build that is passed as input for a job has the same system as the job. - [6]
BK24VA6Q* for git inputs, check latest revision of branch (defaults to master for now), if there is change, only use input if last checkout was > hour ago. - [7]
ENYUXIFPsupport git branches as input in stead of only master branch - [8]
JTRG7RDQadd support for git as jobinput - [9]
3XTHEUMP* Implemented the clone feature.
Change contents
- edit in src/lib/Hydra/Helper/AddBuilds.pm at line 65
- replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 70
my $uri = $value;my $uri = $value; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 72
my $timestamp = time;my $sha256;my $storePath;my $timestamp = time;my $sha256;my $storePath; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 76[4.4623]→[4.4623:4777](∅→∅),[4.4777]→[3.573:640](∅→∅),[3.640]→[4.4844:4899](∅→∅),[4.4844]→[4.4844:4899](∅→∅)
# Some simple caching: don't check a path more than once every N seconds.(my $cachedInput) = $db->resultset('CachedPathInputs')->search({srcpath => $uri, lastseen => {">", $timestamp - 30}},{rows => 1, order_by => "lastseen DESC"});# Some simple caching: don't check a path more than once every N seconds.(my $cachedInput) = $db->resultset('CachedPathInputs')->search({srcpath => $uri, lastseen => {">", $timestamp - 30}},{rows => 1, order_by => "lastseen DESC"}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 81
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {$storePath = $cachedInput->storepath;$sha256 = $cachedInput->sha256hash;$timestamp = $cachedInput->timestamp;} else {if (defined $cachedInput && isValidPath($cachedInput->storepath)) {$storePath = $cachedInput->storepath;$sha256 = $cachedInput->sha256hash;$timestamp = $cachedInput->timestamp;} else { - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 87
print STDERR "copying input ", $name, " from $uri\n";$storePath = `nix-store --add "$uri"`or die "Cannot copy path $uri to the Nix store.\n";chomp $storePath;print STDERR "copying input ", $name, " from $uri\n";$storePath = `nix-store --add "$uri"`or die "Cannot copy path $uri to the Nix store.\n";chomp $storePath; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 92
$sha256 = getStorePathHash $storePath;$sha256 = getStorePathHash $storePath; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 94
($cachedInput) = $db->resultset('CachedPathInputs')->search({srcpath => $uri, sha256hash => $sha256});($cachedInput) = $db->resultset('CachedPathInputs')->search({srcpath => $uri, sha256hash => $sha256}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 97
# Path inputs don't have a natural notion of a "revision",# so we simulate it by using the timestamp that we first# saw this path have this SHA-256 hash. So if the# contents of the path changes, we get a new "revision",# but if it doesn't change (or changes back), we don't get# a new "revision".if (!defined $cachedInput) {txn_do($db, sub {$db->resultset('CachedPathInputs')->create({ srcpath => $uri, timestamp => $timestamp, lastseen => $timestamp, sha256hash => $sha256, storepath => $storePath});# Path inputs don't have a natural notion of a "revision", so# we simulate it by using the timestamp that we first saw this# path have this SHA-256 hash. So if the contents of the path# changes, we get a new "revision", but if it doesn't change# (or changes back), we don't get a new "revision".if (!defined $cachedInput) {txn_do($db, sub {$db->resultset('CachedPathInputs')->create({ srcpath => $uri, timestamp => $timestamp, lastseen => $timestamp, sha256hash => $sha256, storepath => $storePath}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 112
} else {$timestamp = $cachedInput->timestamp;txn_do($db, sub {$cachedInput->update({lastseen => time});});}} else {$timestamp = $cachedInput->timestamp;txn_do($db, sub {$cachedInput->update({lastseen => time});}); - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 118
} - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 120
return{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)};return{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)}; - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 129
- replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 133
my $uri = $value;my $uri = $value; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 135
my $sha256;my $storePath;my $sha256;my $storePath; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 138
# First figure out the last-modified revision of the URI.my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);my $stdout; my $stderr;die "Cannot get head revision of Subversion repository at `$uri':\n$stderr"unless IPC::Run::run(@cmd, \$stdout, \$stderr);my $revision = $stdout; chomp $revision;die unless $revision =~ /^\d+$/;# First figure out the last-modified revision of the URI.my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);my $stdout; my $stderr;die "Cannot get head revision of Subversion repository at `$uri':\n$stderr"unless IPC::Run::run(@cmd, \$stdout, \$stderr);my $revision = $stdout; chomp $revision;die unless $revision =~ /^\d+$/; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 147
(my $cachedInput) = $db->resultset('CachedSubversionInputs')->search({uri => $uri, revision => $revision});(my $cachedInput) = $db->resultset('CachedSubversionInputs')->search({uri => $uri, revision => $revision}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 150
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {$storePath = $cachedInput->storepath;$sha256 = $cachedInput->sha256hash;} else {if (defined $cachedInput && isValidPath($cachedInput->storepath)) {$storePath = $cachedInput->storepath;$sha256 = $cachedInput->sha256hash;} else { - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 155
# Then download this revision into the store.print STDERR "checking out Subversion input ", $name, " from $uri revision $revision\n";$ENV{"NIX_HASH_ALGO"} = "sha256";$ENV{"PRINT_PATH"} = "1";(my $res, $stdout, $stderr) = captureStdoutStderr("nix-prefetch-svn", $uri, $revision);die "Cannot check out Subversion repository `$uri':\n$stderr" unless $res;# Then download this revision into the store.print STDERR "checking out Subversion input ", $name, " from $uri revision $revision\n";$ENV{"NIX_HASH_ALGO"} = "sha256";$ENV{"PRINT_PATH"} = "1";(my $res, $stdout, $stderr) = captureStdoutStderr("nix-prefetch-svn", $uri, $revision);die "Cannot check out Subversion repository `$uri':\n$stderr" unless $res; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 163
($sha256, $storePath) = split ' ', $stdout;($sha256, $storePath) = split ' ', $stdout; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 165
txn_do($db, sub {$db->resultset('CachedSubversionInputs')->create({ uri => $uri, revision => $revision, sha256hash => $sha256, storepath => $storePath});txn_do($db, sub {$db->resultset('CachedSubversionInputs')->create({ uri => $uri, revision => $revision, sha256hash => $sha256, storepath => $storePath}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 173
}} - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 175
return{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => $revision};return{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => $revision}; - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 184
- replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 188
my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);$projectName ||= $project->name;$jobsetName ||= $jobset->name;my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);$projectName ||= $project->name;$jobsetName ||= $jobset->name; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 192
# Pick the most recent successful build of the specified job.(my $prevBuild) = $db->resultset('Builds')->search({ finished => 1, project => $projectName, jobset => $jobsetName, job => $jobName, buildStatus => 0 },{ join => 'resultInfo', order_by => "me.id DESC", rows => 1, where => \ attrsToSQL($attrs, "me.id") });# Pick the most recent successful build of the specified job.(my $prevBuild) = $db->resultset('Builds')->search({ finished => 1, project => $projectName, jobset => $jobsetName, job => $jobName, buildStatus => 0 },{ join => 'resultInfo', order_by => "me.id DESC", rows => 1, where => \ attrsToSQL($attrs, "me.id") }); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 199
if (!defined $prevBuild || !isValidPath($prevBuild->outpath)) {print STDERR "input `", $name, "': no previous build available\n";return undef;}if (!defined $prevBuild || !isValidPath($prevBuild->outpath)) {print STDERR "input `", $name, "': no previous build available\n";return undef;} - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 204
#print STDERR "input `", $name, "': using build ", $prevBuild->id, "\n";#print STDERR "input `", $name, "': using build ", $prevBuild->id, "\n"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 206
my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";my $versionRE = "(?:[A-Za-z0-9\.\-]+)";my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";my $versionRE = "(?:[A-Za-z0-9\.\-]+)"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 209
my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 212
return{ type => "build", storePath => $prevBuild->outpath, id => $prevBuild->id, version => $version};return{ type => "build", storePath => $prevBuild->outpath, id => $prevBuild->id, version => $version}; - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 219
- replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 224
my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);$projectName ||= $project->name;$jobsetName ||= $jobset->name;my ($projectName, $jobsetName, $jobName, $attrs) = parseJobName($value);$projectName ||= $project->name;$jobsetName ||= $jobset->name; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 228
my @latestBuilds = $db->resultset('LatestSucceededForJob')->search({}, {bind => [$projectName, $jobsetName, $jobName]});my @latestBuilds = $db->resultset('LatestSucceededForJob')->search({}, {bind => [$projectName, $jobsetName, $jobName]}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 231
my @validBuilds = ();foreach my $build (@latestBuilds) {if(isValidPath($build->outpath)) {push(@validBuilds,$build);}}my @validBuilds = ();foreach my $build (@latestBuilds) {push(@validBuilds, $build) if isValidPath($build->outpath);}if (scalar(@validBuilds) == 0) {print STDERR "input `", $name, "': no previous build available\n";return undef;}my @inputs = ();foreach my $prevBuild (@validBuilds) {my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";my $versionRE = "(?:[A-Za-z0-9\.\-]+)"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 247
if (scalar(@validBuilds) == 0) {print STDERR "input `", $name, "': no previous build available\n";return undef;}my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;my $input ={ type => "sysbuild", storePath => $prevBuild->outpath, id => $prevBuild->id, version => $version, system => $prevBuild->system};push(@inputs, $input);}return @inputs;} - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 263
my @inputs = ();foreach my $prevBuild (@validBuilds) {my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";my $versionRE = "(?:[A-Za-z0-9\.\-]+)";my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;my $input ={ type => "sysbuild", storePath => $prevBuild->outpath, id => $prevBuild->id, version => $version, system => $prevBuild->system};push(@inputs, $input);}return @inputs;} - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 277
"git", "ls-remote", $uri, $branch);"git", "ls-remote", $uri, $branch); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 286
{uri => $uri, branch => $branch, revision => $revision},{rows => 1});{uri => $uri, branch => $branch, revision => $revision},{rows => 1}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 289
($cachedInput) = $db->resultset('CachedGitInputs')->search({uri => $uri, branch => $branch, lastseen => {">", $timestamp - 3600}},{rows => 1, order_by => "lastseen DESC"});($cachedInput) = $db->resultset('CachedGitInputs')->search({uri => $uri, branch => $branch, lastseen => {">", $timestamp - 3600}},{rows => 1, order_by => "lastseen DESC"}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 296
$sha256 = $cachedInput->sha256hash;$timestamp = $cachedInput->timestamp;$revision = $cachedInput->revision;$sha256 = $cachedInput->sha256hash;$timestamp = $cachedInput->timestamp;$revision = $cachedInput->revision; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 301
# Then download this revision into the store.print STDERR "checking out Git input from $uri";$ENV{"NIX_HASH_ALGO"} = "sha256";$ENV{"PRINT_PATH"} = "1";# Then download this revision into the store.print STDERR "checking out Git input from $uri";$ENV{"NIX_HASH_ALGO"} = "sha256";$ENV{"PRINT_PATH"} = "1"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 306
# Checked out code often wants to be able to run `git# describe', e.g., code that uses Gnulib's `git-version-gen'# script. Thus, we leave `.git' in there. Same for# Subversion (e.g., libgcrypt's build system uses that.)$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";$ENV{"NIX_PREFETCH_SVN_LEAVE_DOT_SVN"} = "1";# Checked out code often wants to be able to run `git# describe', e.g., code that uses Gnulib's `git-version-gen'# script. Thus, we leave `.git' in there. Same for# Subversion (e.g., libgcrypt's build system uses that.)$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";$ENV{"NIX_PREFETCH_SVN_LEAVE_DOT_SVN"} = "1"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 313
# Ask for a "deep clone" to allow "git describe" and similar# tools to work. See# http://thread.gmane.org/gmane.linux.distributions.nixos/3569# for a discussion.$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";# Ask for a "deep clone" to allow "git describe" and similar# tools to work. See# http://thread.gmane.org/gmane.linux.distributions.nixos/3569# for a discussion.$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1"; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 319
(my $res, $stdout, $stderr) = captureStdoutStderr("nix-prefetch-git", $uri, $revision);die "Cannot check out Git repository branch '$branch' at `$uri':\n$stderr" unless $res;(my $res, $stdout, $stderr) = captureStdoutStderr("nix-prefetch-git", $uri, $revision);die "Cannot check out Git repository branch '$branch' at `$uri':\n$stderr" unless $res; - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 323
($sha256, $storePath) = split ' ', $stdout;($cachedInput) = $db->resultset('CachedGitInputs')->search({uri => $uri, branch => $branch, sha256hash => $sha256});($sha256, $storePath) = split ' ', $stdout;($cachedInput) = $db->resultset('CachedGitInputs')->search({uri => $uri, branch => $branch, sha256hash => $sha256}); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 327
if (!defined $cachedInput) {txn_do($db, sub {$db->resultset('CachedGitInputs')->update_or_create({ uri => $uri, branch => $branch, revision => $revision, timestamp => $timestamp, lastseen => $timestamp, sha256hash => $sha256, storepath => $storePath});});} else {$timestamp = $cachedInput->timestamp;txn_do($db, sub {if (!defined $cachedInput) {txn_do($db, sub {$db->resultset('CachedGitInputs')->update_or_create({ uri => $uri, branch => $branch, revision => $revision, timestamp => $timestamp, lastseen => $timestamp, sha256hash => $sha256, storepath => $storePath});});} else {$timestamp = $cachedInput->timestamp;txn_do($db, sub { - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 343
});}});} - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 348[4.1934]→[4.1934:2042](∅→∅),[4.2042]→[4.1239:1270](∅→∅),[4.1270]→[4.2107:2117](∅→∅),[4.2107]→[4.2107:2117](∅→∅)
{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => $revision};{ type => $type, uri => $uri, storePath => $storePath, sha256hash => $sha256, revision => $revision};} - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 356
} - edit in src/lib/Hydra/Helper/AddBuilds.pm at line 360
- replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 366
return fetchInputPath($db, $project, $jobset, $name, $type, $value);return fetchInputPath($db, $project, $jobset, $name, $type, $value); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 369
return fetchInputSVN($db, $project, $jobset, $name, $type, $value);return fetchInputSVN($db, $project, $jobset, $name, $type, $value); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 372
return fetchInputBuild($db, $project, $jobset, $name, $type, $value);return fetchInputBuild($db, $project, $jobset, $name, $type, $value); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 375
return fetchInputSystemBuild($db, $project, $jobset, $name, $type, $value);return fetchInputSystemBuild($db, $project, $jobset, $name, $type, $value); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 378
return fetchInputGit($db, $project, $jobset, $name, $type, $value);return fetchInputGit($db, $project, $jobset, $name, $type, $value); - replacement in src/lib/Hydra/Helper/AddBuilds.pm at line 458
my $validJob = 1;foreach my $arg (@{$job->{arg}}) {my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] ;if($input->{type} eq "sysbuild" && ! ($input->{system} eq $job->{system}) ) {$validJob = 0 ;}}if($validJob) {push(@filteredJobs, $job);}my $validJob = 1;foreach my $arg (@{$job->{arg}}) {my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] ;if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) {$validJob = 0;}}if ($validJob) {push(@filteredJobs, $job);}