Command completion should reveal a number of command-line tools from Hydra:
          <screen>
hydra-build      hydra-init          hydra-update-gc-roots
hydra-eval-jobs  hydra-queue-runner
hydra-evaluator  hydra-server
</screen>

      Command completion should reveal a number of command-line tools
      from Hydra, such as <command>hydra-queue-runner</command>.

#! /var/run/current-system/sw/bin/perl
use strict;
use List::MoreUtils qw(all);
use File::Basename;
use File::stat;
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::PluginHooks;
use Hydra::Model::DB;
use Hydra::Helper::AddBuilds;
use Set::Scalar;
STDOUT->autoflush();
my $db = Hydra::Model::DB->new();
my $config = getHydraConfig();
my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
sub addBuildStepOutputs {
    my ($step) = @_;
    my $drv = derivationFromPath($step->drvpath);
    $step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
        foreach keys %{$drv->{outputs}};
}
sub failDependents {
    my ($drvPath, $status, $errorMsg, $dependents, $startTime, $stopTime, $machine, $propagatedFrom) = @_;
    # Get the referrer closure of $drvPath.
    my $dependentDrvs = Set::Scalar->new(computeFSClosure(1, 0, $drvPath));
    my $time = time();
    txn_do($db, sub {
        for my $d (@dependentBuilds) {
            print STDERR "failing dependent build ", $d->id, " of ", $d->project->name, ":", $d->jobset->name, ":", $d->job->name, "\n";
            $d->update(
                { finished => 1
                , logfile => ''
                , iscachedbuild => 0
                , buildstatus => $drvPath eq $d->drvpath ? 1 : 2
                , starttime => $time
                , stoptime => $time
                , errormsg => undef
                });
            my $step = $d->buildsteps->create(
                { stepnr => nextFreeStepNr($d)
                , type => 0 # = build
                , drvpath => $drvPath
                , busy => 0
                , status => $status
                , starttime => $startTime
                , stoptime => $stopTime
                , errormsg => $errorMsg
                });
            addBuildStepOutputs($step);
        }
    });
}
            push @$dependents, $d;
                , machine => $machine
                , propagatedfrom => $propagatedFrom->id
            next unless $dependentDrvs->has($d->drvpath);
        my @dependentBuilds = $db->resultset('Builds')->search(
            { finished => 0, busy => 0 },
            { columns => ["id", "project", "jobset", "job", "drvpath", "finished", "busy"] });
sub doBuild {
    my ($build) = @_;
    my %outputs;
    $outputs{$_->name} = $_->path foreach $build->buildoutputs->all;
    my $drvPath   = $build->drvpath;
    my $maxsilent = $build->maxsilent;
    my $timeout   = $build->timeout;
    my $isCachedBuild = 1;
    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
    my $startTime = time();
    my $stopTime = undef;
    my $buildStatus = 0; # = succeeded
    my $errormsg = undef;
    unless (all { isValidPath($_) } values(%outputs)) {
        $isCachedBuild = 0;
        # Do the build.
        my $thisBuildFailed = 0;
        my $someBuildFailed = 0;
        # Run Nix to perform the build, and monitor the stderr output
        # to get notifications about specific build steps, the
        # associated log files, etc.
        my $cmd = "nix-store --realise $drvPath " .
            "--timeout $timeout " .
            "--max-silent-time $maxsilent " .
            "--option build-max-log-size 67108864 " .
            "--keep-going --fallback " .
            "--no-build-output --log-type flat --print-build-trace " .
            "--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";
        my $buildStepNr = nextFreeStepNr($build);
        my %buildSteps;
        open OUT, "$cmd |" or die;
        while (<OUT>) {
            unless (/^@\s+/) {
                print STDERR "$_";
                next;
            }
            if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->create(
                        { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
                        , type => 0 # = build
                        , drvpath => $drvPathStep
                        , system => $3
                        , busy => 1
                        });
                    addBuildStepOutputs($step);
                });
            }
            elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                my $machine = $2;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({machine => $machine});
                });
            }
            elsif (/^@\s+build-remote-start\s+(\S+)\s+/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({starttime => time});
                });
            }
            elsif (/^@\s+build-remote-done\s+(\S+)\s+/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({stoptime => time});
                });
            }
            elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({busy => 0, status => 0, stoptime => time});
                    $step->update({stoptime => time}) unless defined $step->update;
                });
            }
            elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
                my $drvPathStep = $1;
                $someBuildFailed = 1;
                $thisBuildFailed = 1 if $drvPath eq $drvPathStep;
                my $errorMsg;
                my $status = 1;
                if ($3 eq "cached") {
                    $status = 8;
                } elsif ($3 eq "timeout") {
                    $status = 7;
                } else {
                    $errorMsg = $4;
                }
                my $now = time;
                my $stepStartTime = $now;
                my $stepStopTime = $now;
                my $machine = "";
                txn_do($db, sub {
                    if ($buildSteps{$drvPathStep}) {
                        my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                        $stepStartTime = $step->starttime;
                        $stepStopTime = $now;
                        $machine = $step->machine;
                        $step->update({busy => 0, status => $status, errormsg => $errorMsg, stoptime => $now});
                    }
                    # Don't write a record if this derivation already
                    # failed previously.  This can happen if this is a
                    # restarted build.
                    elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
                        my $step = $build->buildsteps->create(
                            { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
                            , type => 0 # = build
                            , drvpath => $drvPathStep
                            , busy => 0
                            , status => $status
                            , starttime => $now
                            , stoptime => $now
                            , errormsg => $errorMsg
                            });
                        addBuildStepOutputs($step);
                    }
                });
                # Immediately fail all builds that depend on this derivation.
                failDependents($drvPathStep, $status, $errorMsg, $dependents, $stepStartTime, $stepStopTime, $machine, $build);
            }
            elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->create(
                        { stepnr => ($buildSteps{$path} = $buildStepNr++)
                        , type => 1 # = substitution
                        , busy => 1
                        , starttime => time
                        });
                    # "out" is kinda fake (substitutions don't have named outputs).
                    $step->buildstepoutputs->create({ name => "out", path => $path });
                });
            }
            elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
                    $step->update({busy => 0, status => 0, stoptime => time});
                });
            }
        }
        close OUT;
        my $res = $?;
        $stopTime = time();
        if ($res != 0) {
            if ($thisBuildFailed) { $buildStatus = 1; }
            elsif ($someBuildFailed) { $buildStatus = 2; }
            else { $buildStatus = 3; }
        }
        # Only store the output of running Nix if we have a miscellaneous error.
        $errormsg = undef unless $buildStatus == 3;
    }
  done:
    txn_do($db, sub {
        if ($buildStatus == 0) {
            my $size = 0;
            my $closureSize = 0;
            my $releaseName;
            my @closure = computeFSClosure(0, 0, values %outputs);
            foreach my $path (@closure) {
                my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
                $closureSize += $narSize;
                $size += $narSize if grep { $path eq $_ } values(%outputs);
            }
            foreach my $path (values %outputs) {
                $buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
                $releaseName //= getReleaseName($path);
            }
            $build->update(
                { releasename => $releaseName
                , size => $size
                , closuresize => $closureSize
                });
            addBuildProducts($db, $build);
        }
        $build->update(
            { finished => 1
            , busy => 0
            , locker => ''
            , logfile => ''
            , iscachedbuild => $isCachedBuild
            , buildstatus => $buildStatus
            , starttime => $startTime
            , stoptime => $stopTime // time()
            , errormsg => $errormsg
            });
    });
}
my $buildId = $ARGV[0] or die "syntax: $0 BUILD-ID\n";
print STDERR "performing build $buildId\n";
# Lock the build.  If necessary, steal the lock from the parent
# process (runner.pl).  This is so that if the runner dies, the
# children (i.e. the build.pl instances) can continue to run and won't
# have the lock taken away.
my $build;
txn_do($db, sub {
    $build = $db->resultset('Builds')->find($buildId);
    die "build $buildId doesn't exist\n" unless defined $build;
    die "build $buildId already done\n" if $build->finished;
    if ($build->busy != 0 && $build->locker != getppid) {
        die "build $buildId is already being built";
    }
    $build->update({busy => 1, locker => $$});
    $build->buildsteps->search({busy => 1})->delete;
    $build->buildproducts->delete;
});
die unless $build;
# Do the build.  If it throws an error, unlock the build so that it
# can be retried.
eval {
    doBuild $build;
    print "done\n";
};
if ($@) {
    warn $@;
    txn_do($db, sub {
        $build->update({busy => 0, locker => $$});
    });
}
if ($ENV{'HYDRA_MAIL_TEST'}) {
    my $build = $db->resultset('Builds')->find($buildId);
    notifyBuildFinished(\@plugins, $build, []);
    exit 0;
}
    notifyBuildFinished(\@plugins, $build, $dependents);
        # Mark any remaining active build steps as aborted.
        $build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
            elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
                    $step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
                });
            }
            else {
                print STDERR "unknown Nix trace message: $_";
            }
                        , starttime => time
            $errormsg .= $_;
            "--option print-missing false " .
    if (!isValidPath($drvPath)) {
        $buildStatus = 3;
        $errormsg = "derivation was garbage-collected prior to build";
        goto done;
    }
    my $dependents = [];
}
sub nextFreeStepNr {
    my ($build) = @_;
    my $max = $build->buildsteps->find(
        {}, {select => {max => 'stepnr + 1'}, as => ['max']});
    return (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1;
use Nix::Store;
use Hydra::Plugin;

#! /var/run/current-system/sw/bin/perl
use strict;
use Cwd;
use File::Basename;
use POSIX qw(dup2 :sys_wait_h);
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Model::DB;
use IO::Handle;
use Nix::Store;
use Set::Scalar;
chdir Hydra::Model::DB::getHydraPath or die;
my $db = Hydra::Model::DB->new();
STDOUT->autoflush();
my $lastTime;
#$SIG{CHLD} = 'IGNORE';
sub unlockDeadBuilds {
    # Unlock builds whose building process has died.
    txn_do($db, sub {
        my @builds = $db->resultset('Builds')->search({finished => 0, busy => 1});
        foreach my $build (@builds) {
            my $pid = $build->locker;
            my $unlock = 0;
            if ($pid == $$) {
                if (!defined $lastTime || $build->starttime < $lastTime - 300) {
                    $unlock = 1;
                }
            } elsif (kill(0, $pid) != 1) { # see if we can signal the process
                $unlock = 1;
            }
            if ($unlock) {
                print "build ", $build->id, " pid $pid died, unlocking\n";
                $build->update({ busy => 0, locker => "" });
                $build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
            }
        }
    });
}
sub findBuildDependencyInQueue {
    my ($buildsByDrv, $build) = @_;
    return undef unless isValidPath($build->drvpath);
    my @deps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(0, 0, $build->drvpath);
    return unless scalar @deps > 0;
    foreach my $d (@deps) {
        my $bs = $buildsByDrv->{$d};
        next unless defined $bs;
        return $db->resultset('Builds')->find((@$bs)[0]);
    }
    return undef;
}
sub checkBuilds {
    # print "looking for runnable builds...\n";
    my @buildsStarted;
    my $machines = getMachines;
    txn_do($db, sub {
        # Cache scheduled builds by derivation path to speed up
        # findBuildDependencyInQueue.
        my $buildsByDrv = {};
        push @{$buildsByDrv->{$_->drvpath}}, $_->id
            foreach $db->resultset('Builds')->search({ finished => 0 });
        # Builds in the queue of which a dependency is already building.
        my $blockedBuilds = Set::Scalar->new();
        blockBuilds($buildsByDrv, $blockedBuilds, $_)
            foreach $db->resultset('Builds')->search({ finished => 0, busy => 1 });
        # Get the system types for the runnable builds.
        my @systemTypes = $db->resultset('Builds')->search(
            { finished => 0, busy => 0 },
            { join => ['project'], select => ['system'], as => ['system'], distinct => 1 });
        # Get the total number of scheduling shares.
        my $totalShares = getTotalShares($db) || 1;
        # For each system type, select up to the maximum number of
        # concurrent build for that system type.
        foreach my $system (@systemTypes) {
            # How many builds are already currently executing for this
            # system type?
            my $nrActive = $db->resultset('Builds')->search(
                {finished => 0, busy => 1, system => $system->system})->count;
            (my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system});
            my $max = defined $systemTypeInfo ? $systemTypeInfo->maxconcurrent : $maxConcurrent{$system->system} // 2;
            my $extraAllowed = $max - $nrActive;
            next if $extraAllowed <= 0;
            print STDERR "starting at most $extraAllowed builds for system ${\$system->system}\n";
            j: while ($extraAllowed-- > 0) {
                my @runnableJobsets = $db->resultset('Builds')->search(
                    { finished => 0, busy => 0, system => $system->system },
                    { select => ['project', 'jobset'], distinct => 1 });
                next if @runnableJobsets == 0;
                my $windowSize = 24 * 3600;
                my $totalWindowSize = $windowSize * $max;
                my @res;
                foreach my $b (@runnableJobsets) {
                    my $jobset = $db->resultset('Jobsets')->find($b->get_column('project'), $b->get_column('jobset')) or die;
                    my $timeSpent = $timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')};
                    if (!defined $timeSpent) {
                        $timeSpent = $jobset->builds->search(
                            { },
                            { where => \ ("(finished = 0)")
                            , join => 'buildsteps'
                            , select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
                            , as => "sum" })->single->get_column("sum") // 0;
                        $timeSpent += $jobset->builds->search(
                            { },
                            { where => \ ("(me.stoptime >= " . (time() - $windowSize) . ")")
                            , join => 'buildsteps'
                            , select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
                            , as => "sum" })->single->get_column("sum") // 0;
                        # Add a 30s penalty for each started build.  This
                        # is to account for jobsets that have running
                        # builds but no build steps yet.
                        $timeSpent += $jobset->builds->search({ finished => 0, busy => 1 })->count * $costPerBuild;
                        $timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')} = $timeSpent;
                    }
                    my $share = $jobset->schedulingshares || 1; # prevent division by zero
                    my $used = $timeSpent / ($totalWindowSize * ($share / $totalShares));
                    #printf STDERR "%s:%s: %d s, total used = %.2f%%, share used = %.2f%%\n", $jobset->get_column('project'), $jobset->name, $timeSpent, $timeSpent / $totalWindowSize * 100, $used * 100;
                    push @res, { jobset => $jobset, used => $used };
                }
                foreach my $r (sort { $a->{used} <=> $b->{used} } @res) {
                    my $jobset = $r->{jobset};
                    #print STDERR "selected ", $jobset->get_column('project'), ':', $jobset->name, "\n";
                    # Select the highest-priority build for this jobset.
                    my @builds = $jobset->builds->search(
                        { finished => 0, busy => 0, system => $system->system },
                        { order_by => ["priority DESC", "id"] });
                    foreach my $build (@builds) {
                        # Find a dependency of $build that has no queued
                        # dependencies itself.  This isn't strictly necessary,
                        # but it ensures that Nix builds are done as part of
                        # their corresponding Hydra builds, rather than as a
                        # dependency of some other Hydra build.
                        while (my $dep = findBuildDependencyInQueue($buildsByDrv, $build)) {
                            $build = $dep;
                        }
                        next if $build->busy;
                        printf STDERR "starting build %d (%s:%s:%s) on %s; jobset at %.2f%% of its share\n", 
                            $build->id, $build->project->name, $build->jobset->name, $build->job->name, $build->system, $r->{used} * 100;
                        my $logfile = getcwd . "/logs/" . $build->id;
                        mkdir(dirname $logfile);
                        unlink($logfile);
                        $build->update(
                            { busy => 1
                            , locker => $$
                            , logfile => $logfile
                            });
                        push @buildsStarted, $build;
                        next j;
                    }
                }
                last; # nothing found, give up on this system type
            }
        }
        $lastTime = time();
        $_->update({ starttime => time() }) foreach @buildsStarted;
    });
    # Actually start the builds we just selected.  We need to do this
    # outside the transaction in case it aborts or something.
    foreach my $build (@buildsStarted) {
        my $id = $build->id;
        eval {
            my $logfile = $build->logfile;
            my $child = fork();
            die unless defined $child;
            if ($child == 0) {
                eval {
                    open LOG, ">$logfile" or die "cannot create logfile $logfile";
                    POSIX::dup2(fileno(LOG), 1) or die;
                    POSIX::dup2(fileno(LOG), 2) or die;
                    exec("hydra-build", $id);
                };
                warn "cannot start build $id: $@";
                POSIX::_exit(1);
            }
        };
        if ($@) {
            warn $@;
            txn_do($db, sub {
                $build->update({ busy => 0, locker => $$ });
            });
        }
    }
while (1) {
    eval {
        unlockDeadBuilds;
        checkBuilds;
    };
    warn $@ if $@;
    # print "sleeping...\n";
    sleep(5);
}
        # Clean up zombies.
        while ((waitpid(-1, &WNOHANG)) > 0) { };
}
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--unlock") {
    unlockDeadBuilds;
    exit 0;
}
                        $timeSpentPerJobset->{$jobset->get_column('project')}->{$jobset->name} += $costPerBuild;
                        blockBuilds($buildsByDrv, $blockedBuilds, $build);
                        next if $blockedBuilds->has($build->id);
                my $costPerBuild = 30;
            my $timeSpentPerJobset;
    my %maxConcurrent;
    foreach my $machineName (keys %{$machines}) {
        foreach my $system (@{${$machines}{$machineName}{'systemTypes'}}) {
            $maxConcurrent{$system} = (${$machines}{$machineName}{'maxJobs'} or 0) + ($maxConcurrent{$system} or 0)
        }
    }
sub blockBuilds {
    my ($buildsByDrv, $blockedBuilds, $build) = @_;
    my @rdeps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(1, 0, $build->drvpath);
    foreach my $drv (@rdeps) {
        my $bs = $buildsByDrv->{$drv};
        next if !defined $bs;
        $blockedBuilds->insert($_) foreach @$bs;
    }
}
# Given a build, return an arbitrary queued build on which this build
# depends; or undef if no such build exists.

  hydra-build					\

  hydra-queue-runner				\

                  ;; the time 'hydra-build' attempts to build it.

                  ;; the time 'hydra-queue-runner' attempts to build it.