JAH3UPWAVSHXIPNGL6PROQPZBYZHPJNFONWBDZX4HCX646USZXUQC
X3COYEVISCJI2KSFR47LZU23EZ4LWBX5CAQF7CON4CTCYJDQGH3AC
YRKCK7UBGGIQHWGG2CXCPGSRRWNVWB32W3AIW3WS7WXRKBHY2IAQC
A43SLRSH6OIIEZWLRKLMERX7HOMBICKG7WHELCZYLOJWZ3JFRV2AC
6ZB4CIW66KZMCEBTUWTRRNKQAV5WVPYX4QLFAJT5TTJ3CMS4JMXQC
BLVQGJ4LITFCIR3KAUX3GWAHMYH3XXFC4BDXUSXUVEJFEJ7IMM4AC
NILMMFMYWOTQUABCE56HAHEGMPCOU3Q6H76IFBDUKBICDT7WB5RQC
HPEG2RHVNHOPB5T4ZRXANIRBMVOVY3B5GFETJRYOTDJFVAYH2TQAC
SHYRGAWZFYGRWCRB54TYL2GTU3B4WMQ3J5H4ZHGV27W7PUQN3EWQC
BKOIYITRBRVU4D7XFAZPV5QHAPCBMIO3SBNHYJ6TVT43WR32CTHQC
TM6XBAG2BHYF7HFIO6FRVGBZ6WVGZFPN5VGOA6WNLHZ56ICOOYXAC
3XTHEUMP2ZOMPQWE3S5QWHIHCEJNEXGDPQB3JUVZFPS3RFMY455QC
OOQ2D3KCLFPYNAN253PHWLBQMB6OMO2KYQWQXLTP65SQAYZWQ5LAC
LBNVQXUBEZ45SOTGVXK5UEZXIAIZTJLWZNUYFI4JZ6J65N3KPDVQC
EFWN7JBV7YIHNMCA6ZGFRSHZIUQJ2EX57SWYGVT7ZRJCHPIM2R3QC
J5UVLXOK6EDIL5I7VKWH4V2QDS4DPD7FHRK6XBWSXFRQS4JKXFZQC
BMSQD2ZH3T37RNPRKKKNF5DTJEGBLY4YR2BZDKP7PXJGS3ZBBLWAC
INNOEHO6CTW75YEEGNLUKISQEMOAE4P2C2SEVHEKOVIFHADWY4OAC
Y6AHH4THYQA43V77L43YM42DYRPCMDSWLUV4NKWAQYMPL4NTUIPQC
RXVJFQ5AV3WME4HDVBPSRCALQTXROT4KQPOQVO6KTWTBNZIZZGPAC
5SHCWE7XPQORSOLY7HGAIK2ODKBFPY4KVXRL2W7X6D4WALU544HQC
A22P7HCOGNO3XEQWBNEG2VHTGUB2GKQ75Q6JDMPQNQCLFHE7XBFAC
QBQSQOSYZLHDZ65RLOFEWFAIKHA5UN7V4RQYVXHCWUJDDXXGOZFQC
JI26P3PCQPZIGGHHUYVBCAHVZCFCI3XS7P3742JI2S5TI3TPFGEQC
OSVLMLCQQ52E47CX7TZAIFZSO4VVGRHWJTZY2G23YUPMDCKURTYAC
SGNXIOI4SO56F4YCGVMGZZO3AYXGPDL4FL75RDXAK5Y22IYWPPAQC
C7CXMZ66BZTI5ABDJNPL2JKSHSKDQ645GXUKN7OTQBPEQJ452KQAC
TJK27WSBFOM6LKVCSA7LTTCWUGB4O2AL22PNIDQXS4BFOFCISH2QC
KQS7DSKJHTEHALCPTXMWRX2IBOVC5BWU7RWJ4GP5A2JJSBR7HPKAC
JTRG7RDQXKPSO4ESGDLSVAT5WIFGKDL424MN6YYCVTKCOR2FTXRQC
2WRTOU2Z2R3MWLRZCKK3ZZ2C5LJBH6GNJT77NETVA22Y23DUIX5QC
2WUNXJGWGHQRE24ZJT4VIP35OHO2E4VCKA65TCCKU55BNBIPABCAC
Y5DJHUBMLSYUBCN7HWFGARDK5ZLGMJJCD7I4VDTLASAKN3TZJIKQC
EDRUQ4UK643SUCYMMTS2JS4N3BHARLKRDWRH3QAYWDMWRAMPNAOQC
OXJ4QT6KKJHF6SJKQJAEJEIESREIYUC424OONZUCVZSN5PY6447AC
ARD6Z67TDHK2XMW47CSVOXXF327YGOMYK3EAXTSVVXNDAFQYNO5AC
YFPZ46YK4BOI6VH2H3F757UEGEYONURUAEDAYEIBLRY33PLSSO4AC
HE3GX5IPS6XA7KLNP5WZY2YU27WQS663WSKAL5FHWBSJ5GMM633AC
ENYUXIFPWKNQTG3QVACFS46OLIK7JAMSILO2QWK3TWQKWFZBLOXAC
BK24VA6QHPE6XLXUXPLJMOT75WEXHZ6PPOFLYFUP4U4NYS5FVGWAC
DAKY4N64CDWYVPNGCNSWLI5UQFX45DUFLRLY7KGNYZSJLMP5BBPQC
UO67NQMRXO2CTWPL4ORKOSX5D3GWQGGY2DRLEOCVHKOI7WCHFWBAC
VJ4BZV5RBLNVCV7EC3APZOHSN5HOHHD4RITQOHIMMGUAP5AR3B3AC
7XC62UTYVNK6T7V6BYQKQHSO2OTHX5GUVLCDMK7DY46CWGLVT73QC
OSTNUSGYPOKBUUJBOBDDCQ6DUZC5EXZ4EZTNZXZXTHRN7KYMZH2AC
3I6BNF4SWOQLXZPBESHH4S743QEMOXUJR4IKFRVQ3PO6SUGLPIAAC
6ZSLRO7MT4LN4XVV3GK6NFUHEQP5LBU5CS3ZYIINOOO2MFPHFHSQC
WWUOQ7V4KWGGHG22G22G6FWRHZEDLP3UZHA2C2FOMNT3UU4PFT4AC
VXY434VYUX5ZZNZ7BURKKJ5PLQRJ4S2MAGHGH2CEIMP3JHXO6E6QC
4FWDVNWAUAFDXPXIOBOVM2ZRPZG4LLE4JBKYIXUKR45YN6MEOMXQC
KNJTTZ3ZA7FRNX5YTNDWFLJNAMEA6K4EOR45SJSLQFYWMJQ33ISQC
FXW2UR7FY7NPCEHSCAO4FTZHA47K5YACKHJJVKP6T3V42LJJTLAAC
JZE7DC2FYU4TAGBTMZLXFTISJZ2FTCBLGUDUBKGH4UUIWXRIS3VAC
O25D52TAMOPAK45N4II5XMWOBMPQJNHLW22M37COVY43EKNQBWJAC
M73H45NECYJAVHY635NOBISTSQXEZH3SQMB3OJODGGXZLJVSU3IQC
HQ4QHAOFNFP5CHHZXHQ5VQRPTPPV3RKMMIJJHTPNTGJLO4E4GKWAC
VLS2QTLM4A3SLSSWQMFTD3ODKUXFDCUC4ZMBINO6GQIIH75GDJZAC
P5XCKTFDQHOWNE5ZLPFH27FYZQ7QPCEL33FYR4LTI2JR3IA2ILOAC
VOBOLKL45U33Y3JBB3672X2CNORQWXWRCJOF6DSZRMARLGB4HNTQC
CQTN62OHT4DY35E2MJEG7GFTVNEE5KRDMV6ASBQLBHN7BUDK7WHAC
R4MHON3OOHV63YGRIDHNAV6ZQN7QKAHPRNQUXLIYGC25OUKZ43SQC
QMW24O5S43MYF5ZTBULUEZNDW2FKJ3QP2GL46P6B2SVGSQLVEACAC
CHQEG6WY44VSOTLNH5KDZWJCCHNHRUCX6BEREXUUO56VOKHNSUZAC
V2MLUZRP7K4WGDM54U35F77ICGJMBHDET4SZUDMQNTYTAYKVBFWAC
WQXF2T3DFC2K72OVX46TUQ4IMZNJ5BSBQ7L7AVI7QL54DXCBEKLQC
A63IHCMXH3F4V56HDXJLJVVHKXRSJCJMT2PWXXI2IW3J734J6SGQC
2GK5DOU7ODF4WBSN3QTD3WIO52VTL2LOAXKGCDEMMAQPTEO4A4HAC
BDSD2JLV4V4I52SE7MCZLYNDC4XU27ZNH7TYKLT7CA7YR5WRMQBAC
LZO3C2KIACZ3HN72RBGWWIT5ED4RJMYKI3SAHXT6RIUPHDFL3STQC
PMNWRTGJ4GVSMSSAWSUD57B26PCRAHMZIQ5SIWJIK7A74ENKEQLAC
AFTXA575C6JTVLVXTYJUKQGPLBO3NFORLO5XDSPHNL44HXLRH4TAC
5EQYVRWECBDJORGI5DRIOUEJXSXMRCQNT2562BM4Z4U52LT7JUHAC
LKDWWJLM4CDGAH5C4MNJTZXBJZFV2OJWGR5S7DMMT6AAIGGOL4YAC
4WZQW2N6NJSIKSK7DCOV2YVEG5C45LLUM2FCCZRLISUXUGMBGF6QC
JLDUSNUOOQNL63BOPXIWZOWFRQ5X35RWG33PJB3J3KMR6QR7TN7QC
IK53RV4VGOHLCZGQCCIKPB45M3C7M7YMNBOJFBGZJ4LWIZNU4QNQC
E5DMQRPO5BQ6KNA3C34U5JQQ5ZAZOJE2HTWM3JUVIG42LAVHPHFQC
6KIJX24R5RRDR2UQMUAWHF3N6V6DKKL5URYSLB7IT4J5C3RO4G2AC
ODNCGFQ5FPKFI624BVMLW7PJ2EFJOR3TY66OCZM42UNNTWBCF2TQC
OX6NYJDVCDX2UCJ4QQMTRJVZ535NOMCJGYCHDLCADJPQI7RMCXPQC
JK2QWPH6EOQXGCADV7C5HSYRVWC5M53LAKVN26EMTLY65IKU3AEQC
N22GPKYTOLZLBGTGDATQDVZ4R5APZEAOIA7L32X4UXBH4XNI7MWAC
OVR2RWBIUXNW2XSG63KFL2S3Q7UVTLHOEYO3573LZBY7AMLUKKTQC
POPU2ATH2HHBTGHKRAV3EY2K55P664IARI3YJGLDKVJ6PQPXBQ4AC
sub scmPath {
return Hydra::Model::DB::getHydraPath . "/scm" ;
}
sub getStorePathHash {
my ($storePath) = @_;
my $hash = `nix-store --query --hash $storePath`
or die "cannot get hash of $storePath";
chomp $hash;
die unless $hash =~ /^sha256:(.*)$/;
$hash = $1;
$hash = `nix-hash --to-base16 --type sha256 $hash`
or die "cannot convert hash";
chomp $hash;
return $hash;
}
}
sub fetchInputPath {
my ($db, $project, $jobset, $name, $value) = @_;
my $uri = $value;
my $timestamp = time;
my $sha256;
my $storePath;
# Some simple caching: don't check a path more than once every N seconds.
(my $cachedInput) = $db->resultset('CachedPathInputs')->search(
{srcpath => $uri, lastseen => {">", $timestamp - 30}},
{rows => 1, order_by => "lastseen DESC"});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
} else {
print STDERR "copying input ", $name, " from $uri\n";
$storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store.\n";
chomp $storePath;
$sha256 = getStorePathHash $storePath;
($cachedInput) = $db->resultset('CachedPathInputs')->search(
{srcpath => $uri, sha256hash => $sha256});
# Path inputs don't have a natural notion of a "revision", so
# we simulate it by using the timestamp that we first saw this
# path have this SHA-256 hash. So if the contents of the path
# changes, we get a new "revision", but if it doesn't change
# (or changes back), we don't get a new "revision".
if (!defined $cachedInput) {
txn_do($db, sub {
$db->resultset('CachedPathInputs')->update_or_create(
{ srcpath => $uri
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});
});
} else {
$timestamp = $cachedInput->timestamp;
txn_do($db, sub {
$cachedInput->update({lastseen => time});
});
}
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
};
}
sub fetchInputSVN {
my ($db, $project, $jobset, $name, $value, $checkout) = @_;
# Allow users to specify a revision number next to the URI.
my ($uri, $revision) = split ' ', $value;
my $sha256;
my $storePath;
my $stdout; my $stderr;
unless (defined $revision) {
# First figure out the last-modified revision of the URI.
my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],
"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
IPC::Run::run(@cmd, \$stdout, \$stderr);
die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?;
$revision = $stdout; $revision =~ s/\s*([0-9]+)\s*/$1/sm;
}
die unless $revision =~ /^\d+$/;
# Do we already have this revision in the store?
# !!! This needs to take $checkout into account! Otherwise "svn"
# and "svn-checkout" inputs can get mixed up.
(my $cachedInput) = $db->resultset('CachedSubversionInputs')->search(
{uri => $uri, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
# No, do a checkout. The working copy is reused between
# invocations to speed things up.
my $wcPath = scmPath . "/svn/" . sha256_hex($uri) . "/svn-checkout";
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath);
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
if ($checkout) {
$storePath = addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
}
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
txn_do($db, sub {
$db->resultset('CachedSubversionInputs')->update_or_create(
{ uri => $uri
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
};
sub fetchInputGit {
my ($db, $project, $jobset, $name, $value) = @_;
(my $uri, my $branch, my $deepClone) = split ' ', $value;
$branch = defined $branch ? $branch : "master";
my $timestamp = time;
my $sha256;
my $storePath;
mkpath(scmPath);
my $clonePath = scmPath . "/" . sha256_hex($uri);
my $stdout = ""; my $stderr = ""; my $res;
if (! -d $clonePath) {
# Clone everything and fetch the branch.
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath);
die "error cloning git repo at `$uri':\n$stderr" if $res;
}
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
# This command force the update of the local branch to be in the same as
# the remote branch for whatever the repository state is. This command mirror
# only one branch of the remote repository.
($res, $stdout, $stderr) = captureStdoutStderr(600,
"git", "fetch", "-fu", "origin", "+$branch:$branch");
($res, $stdout, $stderr) = captureStdoutStderr(600,
"git", "fetch", "-fu", "origin") if $res;
die "error fetching latest change from git repo at `$uri':\n$stderr" if $res;
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "rev-parse", "$branch"));
die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res;
my ($revision) = split /\n/, $stdout;
die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout"
unless $revision =~ /^[0-9a-fA-F]+$/;
my $ref = "refs/heads/$branch";
# If deepClone is defined, then we look at the content of the repository
# to determine if this is a top-git branch.
if (defined $deepClone) {
# Checkout the branch to look at its content.
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch");
die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res;
if (-f ".topdeps") {
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
($res, $stdout, $stderr) = captureStdoutStderr(600,
"tg", "remote", "--populate", "origin");
print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res;
}
}
# Some simple caching: don't check a uri/branch/revision more than once.
# TODO: Fix case where the branch is reset to a previous commit.
my $cachedInput ;
($cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, branch => $branch, revision => $revision},
{rows => 1});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
} else {
# Then download this revision into the store.
print STDERR "checking out Git branch $branch from $uri\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
if (defined $deepClone) {
# Checked out code often wants to be able to run `git
# describe', e.g., code that uses Gnulib's `git-version-gen'
# script. Thus, we leave `.git' in there. Same for
# Subversion (e.g., libgcrypt's build system uses that.)
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
# Ask for a "deep clone" to allow "git describe" and similar
# tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion.
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
}
($sha256, $storePath) = split ' ', $stdout;
txn_do($db, sub {
$db->resultset('CachedGitInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
# For convenience in producing readable version names, pass the
# number of commits in the history of this revision (‘revCount’)
# the output of git-describe (‘gitTag’), and the abbreviated
# revision (‘shortRev’).
my $revCount = `git rev-list $revision | wc -l`; chomp $revCount;
die "git rev-list failed" if $? != 0;
my $gitTag = `git describe --always $revision`; chomp $gitTag;
die "git describe failed" if $? != 0;
my $shortRev = `git rev-parse --short $revision`; chomp $shortRev;
die "git rev-parse failed" if $? != 0;
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
, revCount => int($revCount)
, gitTag => $gitTag
, shortRev => $shortRev
};
}
sub fetchInputBazaar {
my ($db, $project, $jobset, $name, $value, $checkout) = @_;
my $uri = $value;
my $sha256;
my $storePath;
my $stdout; my $stderr;
mkpath(scmPath);
my $clonePath = scmPath . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath);
die "error cloning bazaar branch at `$uri':\n$stderr" if $res;
}
chdir $clonePath or die $!;
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull");
die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res;
# First figure out the last-modified revision of the URI.
my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
IPC::Run::run(@cmd, \$stdout, \$stderr);
die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?;
my $revision = $stdout; chomp $revision;
die unless $revision =~ /^\d+$/;
(my $cachedInput) = $db->resultset('CachedBazaarInputs')->search(
{uri => $uri, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
# Then download this revision into the store.
print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = "$checkout";
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"nix-prefetch-bzr", $clonePath, $revision);
die "cannot check out Bazaar branch `$uri':\n$stderr" if $res;
($sha256, $storePath) = split ' ', $stdout;
txn_do($db, sub {
$db->resultset('CachedBazaarInputs')->create(
{ uri => $uri
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
};
}
sub fetchInputHg {
my ($db, $project, $jobset, $name, $value) = @_;
(my $uri, my $id) = split ' ', $value;
$id = defined $id ? $id : "default";
# init local hg clone
my $stdout = ""; my $stderr = "";
mkpath(scmPath);
my $clonePath = scmPath . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"hg", "clone", $uri, $clonePath);
die "error cloning mercurial repo at `$uri':\n$stderr" if $res;
}
# hg pull + check rev
chdir $clonePath or die $!;
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull");
die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res;
(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
"hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}");
die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1;
my ($revision, $revCount, $branch) = split ' ', $stdout;
my $storePath;
my $sha256;
(my $cachedInput) = $db->resultset('CachedHgInputs')->search(
{uri => $uri, branch => $branch, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
print STDERR "checking out Mercurial input from $uri $branch revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"nix-prefetch-hg", $clonePath, $revision);
die "cannot check out Mercurial repository `$uri':\n$stderr" if $res;
($sha256, $storePath) = split ' ', $stdout;
txn_do($db, sub {
$db->resultset('CachedHgInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, branch => $branch
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
, revCount => int($revCount)
};
}
if ($type eq "path") {
push @inputs, fetchInputPath($db, $project, $jobset, $name, $value);
}
elsif ($type eq "svn") {
push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 0);
}
elsif ($type eq "svn-checkout") {
push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 1);
}
elsif ($type eq "build") {
push @inputs, fetchInputBuild($db, $project, $jobset, $name, $value);
if ($type eq "build") {
@inputs = fetchInputBuild($db, $project, $jobset, $name, $value);
elsif ($type eq "git") {
push @inputs, fetchInputGit($db, $project, $jobset, $name, $value);
}
elsif ($type eq "hg") {
push @inputs, fetchInputHg($db, $project, $jobset, $name, $value);
}
elsif ($type eq "bzr") {
push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 0);
}
elsif ($type eq "bzr-checkout") {
push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 1);
}
die "input `" . $name . "' has unknown type `$type'.";
my $found = 0;
foreach my $plugin (@{$plugins}) {
@inputs = $plugin->fetchInput($type, $name, $value);
if (defined $inputs[0]) {
$found = 1;
last;
}
}
die "input `$name' has unknown type `$type'." unless $found;
}
when (["svn", "svn-checkout", "bzr", "bzr-checkout"]) {
push @res, "--arg", $input, (
"{ outPath = builtins.storePath " . $alt->{storePath} . "" .
(defined $alt->{revision} ? "; rev = " . $alt->{revision} . "" : "") .
";}"
);
eval {
local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required
alarm $timeout;
IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr);
alarm 0;
};
if ($@) {
die unless $@ eq "timeout\n"; # propagate unexpected errors
return (-1, "", "timeout\n");
} else {
return ($?, $stdout, $stderr);
}
}
eval {
local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required
alarm $timeout;
IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr);
alarm 0;
};
if ($@) {
die unless $@ eq "timeout\n"; # propagate unexpected errors
return (-1, "", "timeout\n");
} else {
return ($?, $stdout, $stderr);
}
}
package Hydra::Plugin::BazaarInput;
use strict;
use parent 'Hydra::Plugin';
use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'bzr'} = 'Bazaar export';
$inputTypes->{'bzr-checkout'} = 'Bazaar checkout';
}
sub fetchInput {
my ($self, $type, $name, $value) = @_;
return undef if $type ne "bzr" && $type ne "bzr-checkout";
my $uri = $value;
my $sha256;
my $storePath;
my $stdout; my $stderr;
my $cacheDir = getSCMCacheDir . "/bzr";
mkpath($cacheDir);
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath);
die "error cloning bazaar branch at `$uri':\n$stderr" if $res;
}
chdir $clonePath or die $!;
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull");
die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res;
# First figure out the last-modified revision of the URI.
my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
IPC::Run::run(@cmd, \$stdout, \$stderr);
die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?;
my $revision = $stdout; chomp $revision;
die unless $revision =~ /^\d+$/;
(my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
{uri => $uri, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
# Then download this revision into the store.
print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = $type eq "bzr-checkout" ? "1" : "0";
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"nix-prefetch-bzr", $clonePath, $revision);
die "cannot check out Bazaar branch `$uri':\n$stderr" if $res;
($sha256, $storePath) = split ' ', $stdout;
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedBazaarInputs')->create(
{ uri => $uri
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
};
}
1;
package Hydra::Plugin::GitInput;
use strict;
use parent 'Hydra::Plugin';
use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'git'} = 'Git checkout';
}
sub fetchInput {
my ($self, $type, $name, $value) = @_;
return undef if $type ne "git";
(my $uri, my $branch, my $deepClone) = split ' ', $value;
$branch = defined $branch ? $branch : "master";
my $timestamp = time;
my $sha256;
my $storePath;
my $cacheDir = getSCMCacheDir . "/git";
mkpath($cacheDir);
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
my $stdout = ""; my $stderr = ""; my $res;
if (! -d $clonePath) {
# Clone everything and fetch the branch.
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath);
die "error cloning git repo at `$uri':\n$stderr" if $res;
}
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
# This command force the update of the local branch to be in the same as
# the remote branch for whatever the repository state is. This command mirror
# only one branch of the remote repository.
($res, $stdout, $stderr) = captureStdoutStderr(600,
"git", "fetch", "-fu", "origin", "+$branch:$branch");
($res, $stdout, $stderr) = captureStdoutStderr(600,
"git", "fetch", "-fu", "origin") if $res;
die "error fetching latest change from git repo at `$uri':\n$stderr" if $res;
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "rev-parse", "$branch"));
die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res;
my ($revision) = split /\n/, $stdout;
die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout"
unless $revision =~ /^[0-9a-fA-F]+$/;
my $ref = "refs/heads/$branch";
# If deepClone is defined, then we look at the content of the repository
# to determine if this is a top-git branch.
if (defined $deepClone) {
# Checkout the branch to look at its content.
($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch");
die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res;
if (-f ".topdeps") {
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
($res, $stdout, $stderr) = captureStdoutStderr(600,
"tg", "remote", "--populate", "origin");
print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res;
}
}
# Some simple caching: don't check a uri/branch/revision more than once.
# TODO: Fix case where the branch is reset to a previous commit.
my $cachedInput ;
($cachedInput) = $self->{db}->resultset('CachedGitInputs')->search(
{uri => $uri, branch => $branch, revision => $revision},
{rows => 1});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
} else {
# Then download this revision into the store.
print STDERR "checking out Git branch $branch from $uri\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
if (defined $deepClone) {
# Checked out code often wants to be able to run `git
# describe', e.g., code that uses Gnulib's `git-version-gen'
# script. Thus, we leave `.git' in there. Same for
# Subversion (e.g., libgcrypt's build system uses that.)
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
# Ask for a "deep clone" to allow "git describe" and similar
# tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion.
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
}
($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision);
die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res;
($sha256, $storePath) = split ' ', $stdout;
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedGitInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
# For convenience in producing readable version names, pass the
# number of commits in the history of this revision (‘revCount’)
# the output of git-describe (‘gitTag’), and the abbreviated
# revision (‘shortRev’).
my $revCount = `git rev-list $revision | wc -l`; chomp $revCount;
die "git rev-list failed" if $? != 0;
my $gitTag = `git describe --always $revision`; chomp $gitTag;
die "git describe failed" if $? != 0;
my $shortRev = `git rev-parse --short $revision`; chomp $shortRev;
die "git rev-parse failed" if $? != 0;
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
, revCount => int($revCount)
, gitTag => $gitTag
, shortRev => $shortRev
};
}
1;
package Hydra::Plugin::MercurialInput;
use strict;
use parent 'Hydra::Plugin';
use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'hg'} = 'Mercurial checkout';
}
sub fetchInput {
my ($self, $type, $name, $value) = @_;
return undef if $type ne "hg";
(my $uri, my $id) = split ' ', $value;
$id = defined $id ? $id : "default";
# init local hg clone
my $stdout = ""; my $stderr = "";
my $cacheDir = getSCMCacheDir . "/hg";
mkpath($cacheDir);
my $clonePath = $cacheDir . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"hg", "clone", $uri, $clonePath);
die "error cloning mercurial repo at `$uri':\n$stderr" if $res;
}
# hg pull + check rev
chdir $clonePath or die $!;
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull");
die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res;
(my $res1, $stdout, $stderr) = captureStdoutStderr(600,
"hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}");
die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1;
my ($revision, $revCount, $branch) = split ' ', $stdout;
my $storePath;
my $sha256;
(my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
{uri => $uri, branch => $branch, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
print STDERR "checking out Mercurial input from $uri $branch revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
"nix-prefetch-hg", $clonePath, $revision);
die "cannot check out Mercurial repository `$uri':\n$stderr" if $res;
($sha256, $storePath) = split ' ', $stdout;
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedHgInputs')->update_or_create(
{ uri => $uri
, branch => $branch
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, branch => $branch
, storePath => $storePath
, sha256hash => $sha256
, revision => $revision
, revCount => int($revCount)
};
}
1;
package Hydra::Plugin::PathInput;
use strict;
use parent 'Hydra::Plugin';
use POSIX qw(strftime);
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'path'} = 'Local path';
}
sub fetchInput {
my ($self, $type, $name, $value) = @_;
return undef if $type ne "path";
my $uri = $value;
my $timestamp = time;
my $sha256;
my $storePath;
# Some simple caching: don't check a path more than once every N seconds.
(my $cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
{srcpath => $uri, lastseen => {">", $timestamp - 30}},
{rows => 1, order_by => "lastseen DESC"});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
} else {
print STDERR "copying input ", $name, " from $uri\n";
$storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store.\n";
chomp $storePath;
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
{srcpath => $uri, sha256hash => $sha256});
# Path inputs don't have a natural notion of a "revision", so
# we simulate it by using the timestamp that we first saw this
# path have this SHA-256 hash. So if the contents of the path
# changes, we get a new "revision", but if it doesn't change
# (or changes back), we don't get a new "revision".
if (!defined $cachedInput) {
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedPathInputs')->update_or_create(
{ srcpath => $uri
, timestamp => $timestamp
, lastseen => $timestamp
, sha256hash => $sha256
, storepath => $storePath
});
});
} else {
$timestamp = $cachedInput->timestamp;
txn_do($self->{db}, sub {
$cachedInput->update({lastseen => time});
});
}
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp)
};
}
1;
package Hydra::Plugin::SubversionInput;
use strict;
use parent 'Hydra::Plugin';
use Digest::SHA qw(sha256_hex);
use Hydra::Helper::Nix;
use IPC::Run;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
$inputTypes->{'svn'} = 'Subversion export';
$inputTypes->{'svn-checkout'} = 'Subversion checkout';
}
sub fetchInput {
my ($self, $type, $name, $value) = @_;
return undef if $type ne "svn" && $type ne "svn-checkout";
# Allow users to specify a revision number next to the URI.
my ($uri, $revision) = split ' ', $value;
my $sha256;
my $storePath;
my $stdout; my $stderr;
unless (defined $revision) {
# First figure out the last-modified revision of the URI.
my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri],
"|", ["sed", 's/^ *\([0-9]*\).*/\1/']);
IPC::Run::run(@cmd, \$stdout, \$stderr);
die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?;
$revision = int($stdout); $revision =~ s/\s*([0-9]+)\s*/$1/sm;
}
die unless $revision =~ /^\d+$/;
$revision = int($revision);
# Do we already have this revision in the store?
# !!! This needs to take $checkout into account! Otherwise "svn"
# and "svn-checkout" inputs can get mixed up.
(my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
{uri => $uri, revision => $revision});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
# No, do a checkout. The working copy is reused between
# invocations to speed things up.
my $wcPath = getSCMCacheDir . "/svn/" . sha256_hex($uri) . "/svn-checkout";
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
(my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath);
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
if ($type eq "svn-checkout") {
$storePath = addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
}
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
txn_do($self->{db}, sub {
$self->{db}->resultset('CachedSubversionInputs')->update_or_create(
{ uri => $uri
, revision => $revision
, sha256hash => $sha256
, storepath => $storePath
});
});
}
return
{ uri => $uri
, storePath => $storePath
, sha256hash => $sha256
, revNumber => $revision
};
}
1;
inputTypes =
{ "svn" = "Subversion export"
, "svn-checkout" = "Subversion checkout"
, "bzr" = "Bazaar export"
, "bzr-checkout" = "Bazaar checkout"
, "git" = "Git checkout"
, "hg" = "Mercurial checkout"
, "string" = "String value"
, "boolean" = "Boolean"
, "path" = "Local path"
, "build" = "Build output"
, "sysbuild" = "Build output (same system)"
};
my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
foreach my $info_el (@info) {
push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
}
push @{$$inputInfo{$input->name}}, $_
foreach fetchInput($plugins, $db, $project, $jobset, $input->name, $input->type, $alt->value);