UXRNODRJJU7A33F4PG24WOHKTJFI3XQMZGT2MUOBRAH2MPQ7NZ2AC
MREXL5RTGW32MFN2N3LOAYSAYDTVZOIKHZYUYV42WWUT22BZLJEQC
Z4Y3TVEEN6LIGBRCMWVFZYFFE6NKHUSO76RZ3GSPTZEME3CBLUTAC
ZNGKTEAIUI2DJCAWLFL4RFDQGTHCQUAC27JVYS77WTDEI5P5DH4AC
6SGBF7JPJCFTO2N52RLU342XG5LECT7NR3W24U3GLJDRIF7VPHFQC
WMSCRPSC3JXXDLKCY2VMQ4LEQBRQHHKXLLW274MYXISXEKV3WWVQC
6PG5BATLSK2CZP2N4OZIOLW2HFIYQK4SU3CI5PRQNO5EEKXGYFNAC
PHNLYPKB5SJJAHAICS6QEMRNCREPG7SC2MQRLTQMQLWGXM427S3QC
VXTPW6BFORTQYVIM24DONRVWV2VKLPH3GRZIRCT3ODBDTMVXZ6NQC
FUJ3A66L5ENVHQFS3F4W3SBPW64Q6HQ5A5VW36IVC35NUMWZNJCQC
MMUBEIGVTTY7OESMGVEOL7EC7HNTSGFIIOC5C3UHKVFYHJ75EMVAC
IEP2Q6GV6KS3X6M2C6ORW4LXNXX33YW4P7P6EL27VMWO4YV2FUQQC
TSR3RZC4WAL33WIZ6VU5FSSXFHY3DR27365ZDPLFU52P6ZCDRNNAC
AWMM5OGV5EM4ZA3T3SCCU6LMKVJHKI2JANCNA236MD3O7AELE5XAC
ZZYTUBG2HVWNYS5P3FHKK5L5CYN4ZVXPSUE36A27KPHOL6UPJHEAC
533PIYC3GQ4TKUJLKW56CIOD4QRIEWGGXRT4PDA7LN5VFYC4G7OQC
IS7GUIWYGOHEOYMOULIRE45MG7FFSBDN2MYDPKLGZ6PIQ5X4F5XQC
COLZ7V4EBF3RFVVFILN6EWKFKFJUDQF35PMFWIPOKON2LMALT2ZQC
3YHNO5H2IBHEAHVVYKTD4TYWR5AU2MUWYYMMCRSEIL5MDGRXMTIAC
UJOSXBZIZYRUNHMYJGMNG5WCV5DAAWATX3HOR4PSZEDOGKZJHQZAC
E746D2DRXTNOCZKU24OFN5DKTNRCOZSUUUAQV3DNOXZKZG4ZB2BAC
WJ7IKFXL2E3DXW3I35DZDPTQGTW44YJLWIMNCLRULLCH542HRRMQC
QMC6IMJQ3CTMLLEERF5YT7JS3RJW4UJ2BKEJA43WRQ5OL72G6ZMAC
GYPHTT4MZS75JYMQES2DZPV2D47EJW5HTQWYWDYXE7YJ6FFHPAYAC
KBW3FDZ2PK7YCH3ICSHFWP22CWXYAXIPULP6LEN3S3K22UT4WSVQC
EKNK5AHQNJDKGLFPEX6CQXNPVIOWX3JXEP2H5YUZ4UFWCZ3HEJLQC
AMFMXR52JXHUOWWG4VZ5DHQGLF4F7H7XSJRLX5XNEXOCCWYLDQOAC
3PNG7NIBQQURUUPRVQXYL342OT7JUUYOMY2JJNP6YDX7SYJDZMYAC
US27ZTX5HCH53SQN2KFSCV2GL2625XL5J3WCDR7D5HLPA7ZSPX5AC
ERNOO5ZZ5OBBIGIXCLWZHCR73UDUDK7VV5GFWV4HNSWFD4AHCESQC
T4LLYESZ2HUXSLKZ6GNBLVWUVG7R5IDFHYHYO773QIZ6QTOOXR2AC
7YBYT2LQML2PKEO6UO4444AGSASS664UCDXW2YO3ALB7THQHCEBQC
JK2QWPH6EOQXGCADV7C5HSYRVWC5M53LAKVN26EMTLY65IKU3AEQC
D5QIOJGPKQJIYBUCSC3MFJ3TXLPNZ2XMI37GXMFRVRFWWR2VMTFAC
L4AI5YL6CBMVS5GEPME3WIL2CHACMU7PGEKKNKLHHF5SP4ID3LCQC
FV2M6MOTAP4BJMEKU5XUDVEACWEJGEIRCCE2MRY3F6SF2SFOE3MQC
CLXEECMFKLUIN5QBV3BPPPSU6G5UF6MXRSNWA6LXUBNQGSJY4U5QC
PMNWRTGJ4GVSMSSAWSUD57B26PCRAHMZIQ5SIWJIK7A74ENKEQLAC
B72GLND4UQKSGNGIY4FA6HLQNFS42SKRXV5NHTHADUJRU73NELAAC
64K7R4Y6IE4HYE36LMENATBEHOGZ6RNSGJSIZFZGZOLJXM6RRMBAC
5EQYVRWECBDJORGI5DRIOUEJXSXMRCQNT2562BM4Z4U52LT7JUHAC
AFTXA575C6JTVLVXTYJUKQGPLBO3NFORLO5XDSPHNL44HXLRH4TAC
TJK27WSBFOM6LKVCSA7LTTCWUGB4O2AL22PNIDQXS4BFOFCISH2QC
IW2LHCLLCFC6CSM5BECJS3PN2LPBAM4ZJX6NHBZPIRUJTNWHG4EAC
66MEE6QG53NOOFIMZZE7R4TL5SA32V2547ZD3LDT2NPLFNR6Q7CAC
RWIBJ5L4MDV4XQIQXBB45RBVAK2IAK3IMP37YUX2SHW7YDKDAAXAC
FHAVPTZ6GYCZMMPIJN2VPGBACK66NENWCFJMTAOW6464WNCJPVOQC
EUWLW7FYGZK4HKBA73SQDM4FWZGR5F6AGJW6WYZI2ECCIDWF5MRAC
YFPZ46YK4BOI6VH2H3F757UEGEYONURUAEDAYEIBLRY33PLSSO4AC
TULPZ62YXEHXUWGBZMLCLYILEXPQS5ADPT22574BIRFU4CZMBSKAC
6KCP6ODPUDLR2VLWE6LAULMYKI3TWQYEQYZRTYQ4WQX2A5RT6XRQC
WQXF2T3DFC2K72OVX46TUQ4IMZNJ5BSBQ7L7AVI7QL54DXCBEKLQC
BLVQGJ4LITFCIR3KAUX3GWAHMYH3XXFC4BDXUSXUVEJFEJ7IMM4AC
NB2VOKIROXTIIIXKPHFHCGO236KPKXWYDSWUWVJF3XBRHRMUIQ4AC
QUTWJR7PQZ3DBPC4G2AFXOKPPGFZQLE5RLXHAOFTLMXIT2QPF4TQC
BOFOHCPK7UVE3TUEL4NNOQXN2EQGN6N3ITJWTCOJK5Q6MSHYSOSQC
OG7BEM57MXPCG56IT5GODPFG77KZXWPXZ7OVZPWQQ56CRJ7KUE2QC
MNZ67UXOZ427J4FLNA5STC6EB567WU65PJE7B5IOLM4UICU3RFNQC
4SJPAAJXSRMY4UWTJPXAHTLJT2CHHQUEZDNBAEOCMHOAOPEWPBAAC
SX5XYD6IDS4ZCHRMX6NRLSOSGXUE6WYYA5CMKXMNGQWTMXXLENZQC
JGLE5BRNJTMCZFZ33G2SGOQ4F3CIZAPMU3IXCNMR5NNSG5EJLX6QC
5O6E5SU5YQ4JIXLWGHTSAJO4UAMRN65GAZLYMVBMRLFS2F6X7AKAC
UCVYVMXBX5X5CSFUCU2JMG7EIOCMKZ4GY4KT6E5HDZD2ZVJ3UVYQC
IUCHXUJPTNTLOMKTLVDFUKFYLWZHE7MHRZWU3F35RNJR7XM4XJTAC
L2E6EVE2RVFVDCUNRJ4CZYSQNS2DZUA5DTBETHBDUQUV2KQQRAOQC
ZTQEU5QSAJYC66QQABJLVR7EEZWEIW6UMQMSIEP2KRZH5PGEB53QC
BD3GRK4BQUGRQGTAILUITIH3RRSVUH6AKINVMMKYLBAAHAAUSB4AC
37R34XJOGMP3E3DD4RGGRX5LBW2LGB6J3E6XOXLWSU2QEIWBEM4QC
4LR7CSBT5VKBZGKSJIV7N4NX4L3QAZSTNSXDJY53NPZDWLPSP6DQC
NLJJZVHO3UXBURL2P7VGGCVUOMKFUYT3UX5JXQU3FFFAHUGEKO3AC
RBHHV7P7MMFXUNWCJFNYKLGVLG2NQEDUKJ2IUCZ3WFMHH3SZRUHQC
CLJQCY2XHIDNNMFBJ5PK3GQEN6RFALEFKXBJRWZPEIKR4PR5ZQ5AC
A32UT34UUGWVULFYQWSJWIIRCL3JE65PWRMJPEVFFR4MEJTAXQKAC
VJP6O6WATHE7TKODVNIK3GVTUYZTOIELNSI7BXRKS2NVK7MFR22QC
2KDHXY52BVIJPKPA6GYGAXJQF4PXCT2RYRMKRXDG4BCJ2MFF5WFQC
2IEFMER5QDRKX5KMGFKS4G4T7SWHEUKKLEZCHWV2X7PF4UVHBRJQC
XYBSTHMJ3NPHX2H34VPZFU3JBS7AHZAIYS3SCIMPQ6ERJKUPW64AC
G5A7TZVIRTVTI7P5SESOC567QEYSM4SKBZJNYHBXNCTUV53WSZIAC
MPGVCHVFX673NHLU3TXNLAVEVP4PJSMVBMZOQQKD53N6KAG6CTLQC
A63IHCMXH3F4V56HDXJLJVVHKXRSJCJMT2PWXXI2IW3J734J6SGQC
LZO3C2KIACZ3HN72RBGWWIT5ED4RJMYKI3SAHXT6RIUPHDFL3STQC
D7NXMCONBBKA5GLRTUGI3XOTEK3TXHN26UQT4Z5CBIG7CILIQSEAC
3E6IP3R3JGH76PNGG7RCADV65KOV24HQXPXNLVVYIQ46AVYJRG3AC
EFWN7JBV7YIHNMCA6ZGFRSHZIUQJ2EX57SWYGVT7ZRJCHPIM2R3QC
LQNBKF3DKVYK3NPDGK7OLOTIWIF4HR3EKLRYNLZJB4G3NMIVBCEAC
GAIBDEZZPZ52ASD3GK3WUYG3NOB27MLQ5GGIGKLAD6JQ4UATTT3AC
AS5PAYLIRJWB4JGU6N7WKILONGDIGCKNKS4ZSLRYCNGMUAP3AQSAC
EX4FXA5TRG4UK4NASBDNYKW242ROLB42KE6S4I5AKV56LW6CBLHAC
K3EAQY3XGC7FBENSSILPXZEKBDTZEOKCBJKZJRX3GABBGZIJIJ3AC
G3IUM7VLQ4B3N3432X6EMKUJO6NYMXSS5AAYJ6CX4GLXCYXSUWDQC
3BKF6P72K4YGWSQD5PO6PKNM3KXGFLONB3SONKRZR4DJHH3W3PSAC
DGZE7ZRAFD5W2NZILJB34B4EQKI274YBFW2SXWODEPW2GJ6KE62AC
HE3GX5IPS6XA7KLNP5WZY2YU27WQS663WSKAL5FHWBSJ5GMM633AC
UVNQPK3TO3WNRYCHV66SECURHJEKYSDLREKG3T2JVC5YLE2EHA5QC
6US6LEC7V7LUTMPXOWGU7D4PQAFOG2AKNW4VBPSENNBEDMTBJCVQC
A22P7HCOGNO3XEQWBNEG2VHTGUB2GKQ75Q6JDMPQNQCLFHE7XBFAC
HPEG2RHVNHOPB5T4ZRXANIRBMVOVY3B5GFETJRYOTDJFVAYH2TQAC
5DSF5KWYCNFCWHDWAJDWQOFQML2AJIV7CESESP7HVTU4LVEEEYEQC
CMU3YKOUUB6VNP54NVYUKFHI3A46YAAANNYOZSHCUJIQOQEZ3GCAC
CXRCPDSQEWHSZNCJIORUAINH5I2FFFQ7AVUQOB62REGROLROPE7QC
KA45EBF5GVBBFQ2E6GSMDJ5ORSOJ7YP5EHAHLUZB226F7WNO6KYQC
FHF6IZJQPUQHY5QWQYRPZVDBRLHREWRHGNKVQDT7F3GQKKLZXJKQC
MOX7XJ2E3XISXA7V7T4W6GEAGECGWBZ4PYSLTYBVVR4VAKOI33CQC
D3DIBMOKXK2E65267BEEWQL4S4NSHGZBCY7YTU34JSEPZ7AKNBRQC
WHAFVCEIHCWHKY2UVRGRQORERUAK6SZJI7ZENA367ZX4EXDIQMDQC
D6YQQQCNJT7RJ4KF5JN76NM5HT4FVROBGSNB26LYPRYVLEG576MAC
IJPTEKRF3R2HDCUFK3TCVCVVLSF6JSO57RY4DRX3RQPIGOWDB2TQC
7DWCXNC72OO3M4YGNVV4KM775CUSPBHEFFNMRIA2XAZTQCXYHBRQC
OV7F5M3EQBESXVE3GGPXCTEBF2KZH6CUM6ZTXEF72HDSKRLUIZRAC
I7UELKBVDQOL47ZCBLDOLP3KUEFDNCWUPL4K4IG6FDRVVHOBWQ4QC
YTESD75ZEM5RKKKQGS22LCJBGLDJIGYDEHRRW4XD7IUPDXLOCWLQC
J4AJQ6RXK3DCKTCRXNBWG3Y2K6PRPJU5AY4UPQIGV3L6LXMOS3NAC
DQD7JMSUAPXKASPXUDXY4LCE6QCAZSGWNQYTOKNIQE6RRSRP4KGAC
WM5WAST3S67EWKA52NVQ2V55XO6E4WRPAOUGSQGA7XDX46ARX4OQC
CQJX3RGUZ6DJ6PDUWRA4SCUFFDBS5AE74GVIN2ZPL2BV6HJT6VXQC
ULHEOJO2ZUIUWD3LLPRQ4KNPK3O4IBTTW4XUVGNLNUG6IG7KFDHAC
GEADFVZ5LXXFIE3VIP4UJ4AEI2VX57DXER47JA4IHH5BG3QNPAEAC
Y6AHH4THYQA43V77L43YM42DYRPCMDSWLUV4NKWAQYMPL4NTUIPQC
DTXTS7LNTY43YDI5FXVDEJZ75JMR4OW6WS7ZBUXXM3L242VEIIWQC
CURCK6C2RZYAHNQPF4KBC36SVYYWZOGNLBYPXCQ3WABHEIS7TQPQC
YEXD7CBKPTMRCYQFNJFWXJUFOAKXNCVVYYJW3PHXQ5CX53Y5CUGQC
TPSCSZKXKQAMSLMRACXSHWQ4OY7HPSJSJ4KTJY6HLHZQAJ5MDRZAC
WTUBAA5QTHPVRQYEX2QPFFA6XO3BFD4O7JDCR5LXMJVPIR5EI3LAC
LSPRR4TK5XL7FSO32Y54BWQOHJLAFJYEFWBE5TIXA2PMC42SRWFAC
PYTQMQKDYAR66MOX2V7ZHLCJKHASTLTEVVJXCEY3LMRDXCCXAU4QC
SA5ZZ3I4SPTEEPJ5YE6JJA2DUPD4UTTC3APBNQL6XZW6PXYJRP6QC
NREF6YOAKK2QVNXGU7WHD3KSDNCCIAQW3GQ5KOANEXFBWDPUHWXAC
YTIDBFGUR65L6D6OKGQWTKHJJQK3NCC6WDNQFTA7E6EATVOX4Y5AC
QZLMDKMUE7CJ4IMNIM5VJUGF54XXUYETOVI6VTCZ627ADOM4PPKAC
RNNLIVYYCD6HQTY6WSKOYJ6FTWY7SFLZAZ42IEQM3JFWP3GOYQMQC
X3YLTWJAF7QLQGUVGDYALE75W4SJIOYZEFSGG726XRPGN576TWVQC
TFLAR4KANBOKVE63435RE4A5JHEDYD2ARZIECI64DKVOSBHJULGAC
2PWOXJTXWLCCPMW22EVJ7Q4BOBNTU62WJQI5ZPOVUPVW74BDK3NQC
TWVSALRLHGSY6SHLMLT53K2A5SUCPU3SNFPZ2KWYOLDQUJBSZ2OAC
D7TT2BNKYQH6KC2OLMDZKGQPUTPCWA5EBJZHQ7UMUOA7AJMUBGFAC
Y6H7Y3OTXVLF6PJ5BR6WXS7KVS4VMTUKRYVM6FALKMY3DCBYEJSQC
QMW24O5S43MYF5ZTBULUEZNDW2FKJ3QP2GL46P6B2SVGSQLVEACAC
Command completion should reveal a number of command-line tools from Hydra:
<screen>
hydra-build hydra-init hydra-update-gc-roots
hydra-eval-jobs hydra-queue-runner
hydra-evaluator hydra-server
</screen>
Command completion should reveal a number of command-line tools
from Hydra, such as <command>hydra-queue-runner</command>.
#! /var/run/current-system/sw/bin/perl
use strict;
use List::MoreUtils qw(all);
use File::Basename;
use File::stat;
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::PluginHooks;
use Hydra::Model::DB;
use Hydra::Helper::AddBuilds;
use Set::Scalar;
STDOUT->autoflush();
my $db = Hydra::Model::DB->new();
my $config = getHydraConfig();
my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
sub addBuildStepOutputs {
my ($step) = @_;
my $drv = derivationFromPath($step->drvpath);
$step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
foreach keys %{$drv->{outputs}};
}
sub failDependents {
my ($drvPath, $status, $errorMsg, $dependents, $startTime, $stopTime, $machine, $propagatedFrom) = @_;
# Get the referrer closure of $drvPath.
my $dependentDrvs = Set::Scalar->new(computeFSClosure(1, 0, $drvPath));
my $time = time();
txn_do($db, sub {
for my $d (@dependentBuilds) {
print STDERR "failing dependent build ", $d->id, " of ", $d->project->name, ":", $d->jobset->name, ":", $d->job->name, "\n";
$d->update(
{ finished => 1
, logfile => ''
, iscachedbuild => 0
, buildstatus => $drvPath eq $d->drvpath ? 1 : 2
, starttime => $time
, stoptime => $time
, errormsg => undef
});
my $step = $d->buildsteps->create(
{ stepnr => nextFreeStepNr($d)
, type => 0 # = build
, drvpath => $drvPath
, busy => 0
, status => $status
, starttime => $startTime
, stoptime => $stopTime
, errormsg => $errorMsg
});
addBuildStepOutputs($step);
}
});
}
push @$dependents, $d;
, machine => $machine
, propagatedfrom => $propagatedFrom->id
next unless $dependentDrvs->has($d->drvpath);
my @dependentBuilds = $db->resultset('Builds')->search(
{ finished => 0, busy => 0 },
{ columns => ["id", "project", "jobset", "job", "drvpath", "finished", "busy"] });
sub doBuild {
my ($build) = @_;
my %outputs;
$outputs{$_->name} = $_->path foreach $build->buildoutputs->all;
my $drvPath = $build->drvpath;
my $maxsilent = $build->maxsilent;
my $timeout = $build->timeout;
my $isCachedBuild = 1;
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
my $startTime = time();
my $stopTime = undef;
my $buildStatus = 0; # = succeeded
my $errormsg = undef;
unless (all { isValidPath($_) } values(%outputs)) {
$isCachedBuild = 0;
# Do the build.
my $thisBuildFailed = 0;
my $someBuildFailed = 0;
# Run Nix to perform the build, and monitor the stderr output
# to get notifications about specific build steps, the
# associated log files, etc.
my $cmd = "nix-store --realise $drvPath " .
"--timeout $timeout " .
"--max-silent-time $maxsilent " .
"--option build-max-log-size 67108864 " .
"--keep-going --fallback " .
"--no-build-output --log-type flat --print-build-trace " .
"--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";
my $buildStepNr = nextFreeStepNr($build);
my %buildSteps;
open OUT, "$cmd |" or die;
while (<OUT>) {
unless (/^@\s+/) {
print STDERR "$_";
next;
}
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, system => $3
, busy => 1
});
addBuildStepOutputs($step);
});
}
elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
my $machine = $2;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({machine => $machine});
});
}
elsif (/^@\s+build-remote-start\s+(\S+)\s+/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({starttime => time});
});
}
elsif (/^@\s+build-remote-done\s+(\S+)\s+/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({stoptime => time});
});
}
elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({busy => 0, status => 0, stoptime => time});
$step->update({stoptime => time}) unless defined $step->update;
});
}
elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
my $drvPathStep = $1;
$someBuildFailed = 1;
$thisBuildFailed = 1 if $drvPath eq $drvPathStep;
my $errorMsg;
my $status = 1;
if ($3 eq "cached") {
$status = 8;
} elsif ($3 eq "timeout") {
$status = 7;
} else {
$errorMsg = $4;
}
my $now = time;
my $stepStartTime = $now;
my $stepStopTime = $now;
my $machine = "";
txn_do($db, sub {
if ($buildSteps{$drvPathStep}) {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$stepStartTime = $step->starttime;
$stepStopTime = $now;
$machine = $step->machine;
$step->update({busy => 0, status => $status, errormsg => $errorMsg, stoptime => $now});
}
# Don't write a record if this derivation already
# failed previously. This can happen if this is a
# restarted build.
elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, busy => 0
, status => $status
, starttime => $now
, stoptime => $now
, errormsg => $errorMsg
});
addBuildStepOutputs($step);
}
});
# Immediately fail all builds that depend on this derivation.
failDependents($drvPathStep, $status, $errorMsg, $dependents, $stepStartTime, $stepStopTime, $machine, $build);
}
elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$path} = $buildStepNr++)
, type => 1 # = substitution
, busy => 1
, starttime => time
});
# "out" is kinda fake (substitutions don't have named outputs).
$step->buildstepoutputs->create({ name => "out", path => $path });
});
}
elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 0, stoptime => time});
});
}
}
close OUT;
my $res = $?;
$stopTime = time();
if ($res != 0) {
if ($thisBuildFailed) { $buildStatus = 1; }
elsif ($someBuildFailed) { $buildStatus = 2; }
else { $buildStatus = 3; }
}
# Only store the output of running Nix if we have a miscellaneous error.
$errormsg = undef unless $buildStatus == 3;
}
done:
txn_do($db, sub {
if ($buildStatus == 0) {
my $size = 0;
my $closureSize = 0;
my $releaseName;
my @closure = computeFSClosure(0, 0, values %outputs);
foreach my $path (@closure) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
$closureSize += $narSize;
$size += $narSize if grep { $path eq $_ } values(%outputs);
}
foreach my $path (values %outputs) {
$buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
$releaseName //= getReleaseName($path);
}
$build->update(
{ releasename => $releaseName
, size => $size
, closuresize => $closureSize
});
addBuildProducts($db, $build);
}
$build->update(
{ finished => 1
, busy => 0
, locker => ''
, logfile => ''
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime // time()
, errormsg => $errormsg
});
});
}
my $buildId = $ARGV[0] or die "syntax: $0 BUILD-ID\n";
print STDERR "performing build $buildId\n";
# Lock the build. If necessary, steal the lock from the parent
# process (runner.pl). This is so that if the runner dies, the
# children (i.e. the build.pl instances) can continue to run and won't
# have the lock taken away.
my $build;
txn_do($db, sub {
$build = $db->resultset('Builds')->find($buildId);
die "build $buildId doesn't exist\n" unless defined $build;
die "build $buildId already done\n" if $build->finished;
if ($build->busy != 0 && $build->locker != getppid) {
die "build $buildId is already being built";
}
$build->update({busy => 1, locker => $$});
$build->buildsteps->search({busy => 1})->delete;
$build->buildproducts->delete;
});
die unless $build;
# Do the build. If it throws an error, unlock the build so that it
# can be retried.
eval {
doBuild $build;
print "done\n";
};
if ($@) {
warn $@;
txn_do($db, sub {
$build->update({busy => 0, locker => $$});
});
}
if ($ENV{'HYDRA_MAIL_TEST'}) {
my $build = $db->resultset('Builds')->find($buildId);
notifyBuildFinished(\@plugins, $build, []);
exit 0;
}
notifyBuildFinished(\@plugins, $build, $dependents);
# Mark any remaining active build steps as aborted.
$build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
});
}
else {
print STDERR "unknown Nix trace message: $_";
}
, starttime => time
$errormsg .= $_;
"--option print-missing false " .
if (!isValidPath($drvPath)) {
$buildStatus = 3;
$errormsg = "derivation was garbage-collected prior to build";
goto done;
}
my $dependents = [];
}
sub nextFreeStepNr {
my ($build) = @_;
my $max = $build->buildsteps->find(
{}, {select => {max => 'stepnr + 1'}, as => ['max']});
return (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1;
use Nix::Store;
use Hydra::Plugin;
#! /var/run/current-system/sw/bin/perl
use strict;
use Cwd;
use File::Basename;
use POSIX qw(dup2 :sys_wait_h);
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Model::DB;
use IO::Handle;
use Nix::Store;
use Set::Scalar;
chdir Hydra::Model::DB::getHydraPath or die;
my $db = Hydra::Model::DB->new();
STDOUT->autoflush();
my $lastTime;
#$SIG{CHLD} = 'IGNORE';
sub unlockDeadBuilds {
# Unlock builds whose building process has died.
txn_do($db, sub {
my @builds = $db->resultset('Builds')->search({finished => 0, busy => 1});
foreach my $build (@builds) {
my $pid = $build->locker;
my $unlock = 0;
if ($pid == $$) {
if (!defined $lastTime || $build->starttime < $lastTime - 300) {
$unlock = 1;
}
} elsif (kill(0, $pid) != 1) { # see if we can signal the process
$unlock = 1;
}
if ($unlock) {
print "build ", $build->id, " pid $pid died, unlocking\n";
$build->update({ busy => 0, locker => "" });
$build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
}
}
});
}
sub findBuildDependencyInQueue {
my ($buildsByDrv, $build) = @_;
return undef unless isValidPath($build->drvpath);
my @deps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(0, 0, $build->drvpath);
return unless scalar @deps > 0;
foreach my $d (@deps) {
my $bs = $buildsByDrv->{$d};
next unless defined $bs;
return $db->resultset('Builds')->find((@$bs)[0]);
}
return undef;
}
sub checkBuilds {
# print "looking for runnable builds...\n";
my @buildsStarted;
my $machines = getMachines;
txn_do($db, sub {
# Cache scheduled builds by derivation path to speed up
# findBuildDependencyInQueue.
my $buildsByDrv = {};
push @{$buildsByDrv->{$_->drvpath}}, $_->id
foreach $db->resultset('Builds')->search({ finished => 0 });
# Builds in the queue of which a dependency is already building.
my $blockedBuilds = Set::Scalar->new();
blockBuilds($buildsByDrv, $blockedBuilds, $_)
foreach $db->resultset('Builds')->search({ finished => 0, busy => 1 });
# Get the system types for the runnable builds.
my @systemTypes = $db->resultset('Builds')->search(
{ finished => 0, busy => 0 },
{ join => ['project'], select => ['system'], as => ['system'], distinct => 1 });
# Get the total number of scheduling shares.
my $totalShares = getTotalShares($db) || 1;
# For each system type, select up to the maximum number of
# concurrent build for that system type.
foreach my $system (@systemTypes) {
# How many builds are already currently executing for this
# system type?
my $nrActive = $db->resultset('Builds')->search(
{finished => 0, busy => 1, system => $system->system})->count;
(my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system});
my $max = defined $systemTypeInfo ? $systemTypeInfo->maxconcurrent : $maxConcurrent{$system->system} // 2;
my $extraAllowed = $max - $nrActive;
next if $extraAllowed <= 0;
print STDERR "starting at most $extraAllowed builds for system ${\$system->system}\n";
j: while ($extraAllowed-- > 0) {
my @runnableJobsets = $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system },
{ select => ['project', 'jobset'], distinct => 1 });
next if @runnableJobsets == 0;
my $windowSize = 24 * 3600;
my $totalWindowSize = $windowSize * $max;
my @res;
foreach my $b (@runnableJobsets) {
my $jobset = $db->resultset('Jobsets')->find($b->get_column('project'), $b->get_column('jobset')) or die;
my $timeSpent = $timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')};
if (!defined $timeSpent) {
$timeSpent = $jobset->builds->search(
{ },
{ where => \ ("(finished = 0)")
, join => 'buildsteps'
, select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
, as => "sum" })->single->get_column("sum") // 0;
$timeSpent += $jobset->builds->search(
{ },
{ where => \ ("(me.stoptime >= " . (time() - $windowSize) . ")")
, join => 'buildsteps'
, select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
, as => "sum" })->single->get_column("sum") // 0;
# Add a 30s penalty for each started build. This
# is to account for jobsets that have running
# builds but no build steps yet.
$timeSpent += $jobset->builds->search({ finished => 0, busy => 1 })->count * $costPerBuild;
$timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')} = $timeSpent;
}
my $share = $jobset->schedulingshares || 1; # prevent division by zero
my $used = $timeSpent / ($totalWindowSize * ($share / $totalShares));
#printf STDERR "%s:%s: %d s, total used = %.2f%%, share used = %.2f%%\n", $jobset->get_column('project'), $jobset->name, $timeSpent, $timeSpent / $totalWindowSize * 100, $used * 100;
push @res, { jobset => $jobset, used => $used };
}
foreach my $r (sort { $a->{used} <=> $b->{used} } @res) {
my $jobset = $r->{jobset};
#print STDERR "selected ", $jobset->get_column('project'), ':', $jobset->name, "\n";
# Select the highest-priority build for this jobset.
my @builds = $jobset->builds->search(
{ finished => 0, busy => 0, system => $system->system },
{ order_by => ["priority DESC", "id"] });
foreach my $build (@builds) {
# Find a dependency of $build that has no queued
# dependencies itself. This isn't strictly necessary,
# but it ensures that Nix builds are done as part of
# their corresponding Hydra builds, rather than as a
# dependency of some other Hydra build.
while (my $dep = findBuildDependencyInQueue($buildsByDrv, $build)) {
$build = $dep;
}
next if $build->busy;
printf STDERR "starting build %d (%s:%s:%s) on %s; jobset at %.2f%% of its share\n",
$build->id, $build->project->name, $build->jobset->name, $build->job->name, $build->system, $r->{used} * 100;
my $logfile = getcwd . "/logs/" . $build->id;
mkdir(dirname $logfile);
unlink($logfile);
$build->update(
{ busy => 1
, locker => $$
, logfile => $logfile
});
push @buildsStarted, $build;
next j;
}
}
last; # nothing found, give up on this system type
}
}
$lastTime = time();
$_->update({ starttime => time() }) foreach @buildsStarted;
});
# Actually start the builds we just selected. We need to do this
# outside the transaction in case it aborts or something.
foreach my $build (@buildsStarted) {
my $id = $build->id;
eval {
my $logfile = $build->logfile;
my $child = fork();
die unless defined $child;
if ($child == 0) {
eval {
open LOG, ">$logfile" or die "cannot create logfile $logfile";
POSIX::dup2(fileno(LOG), 1) or die;
POSIX::dup2(fileno(LOG), 2) or die;
exec("hydra-build", $id);
};
warn "cannot start build $id: $@";
POSIX::_exit(1);
}
};
if ($@) {
warn $@;
txn_do($db, sub {
$build->update({ busy => 0, locker => $$ });
});
}
}
while (1) {
eval {
unlockDeadBuilds;
checkBuilds;
};
warn $@ if $@;
# print "sleeping...\n";
sleep(5);
}
# Clean up zombies.
while ((waitpid(-1, &WNOHANG)) > 0) { };
}
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--unlock") {
unlockDeadBuilds;
exit 0;
}
$timeSpentPerJobset->{$jobset->get_column('project')}->{$jobset->name} += $costPerBuild;
blockBuilds($buildsByDrv, $blockedBuilds, $build);
next if $blockedBuilds->has($build->id);
my $costPerBuild = 30;
my $timeSpentPerJobset;
my %maxConcurrent;
foreach my $machineName (keys %{$machines}) {
foreach my $system (@{${$machines}{$machineName}{'systemTypes'}}) {
$maxConcurrent{$system} = (${$machines}{$machineName}{'maxJobs'} or 0) + ($maxConcurrent{$system} or 0)
}
}
sub blockBuilds {
my ($buildsByDrv, $blockedBuilds, $build) = @_;
my @rdeps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(1, 0, $build->drvpath);
foreach my $drv (@rdeps) {
my $bs = $buildsByDrv->{$drv};
next if !defined $bs;
$blockedBuilds->insert($_) foreach @$bs;
}
}
# Given a build, return an arbitrary queued build on which this build
# depends; or undef if no such build exists.