Basic remote building

[?]

Jun 9, 2015, 12:21 PM

5AIYUMTBY6TFQTBRP3MJ2PYWUMRF57I77NIVWYE74UMEVQMBWZVQC

Dependencies

[2] O776XDS2 Make getDrvLogPath work with both bucketed and non-bucketed nix logs.
[3] YZAI5GQU Implement a database connection pool
[4] NJJ7H64S Very basic multi-threaded queue runner
[5] T2EIYJNG On SIGINT, shut down the builder threads
[6] 24BMQDZA Start of single-process hydra-queue-runner
[7] 62MQPRXC Pass null values to libpqxx properly
[8] ZH6B56XR Try harder to find build logs
[9] Y6AHH4TH Remove the logfile and logSize columns from the database
[10] ENXUSMSV Make concurrency more robust
[*] 2GK5DOU7 * Downloading closures.

Change contents

replacement in src/hydra-queue-runner/Makefile.am at line 3

[4.255]→[4.255:322](∅→∅)

hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc

[4.255]

[4.322]

hydra_queue_runner_SOURCES = hydra-queue-runner.cc build-result.cc build-remote.cc
file addition: build-remote.cc (----------)

[4.187]

#include <algorithm>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include "build-remote.hh"

#include "util.hh"
#include "misc.hh"
#include "serve-protocol.hh"
#include "worker-protocol.hh"

using namespace nix;

struct Child
{
Pid pid;
AutoCloseFD to, from;
};

static void openConnection(const string & sshName, const string & sshKey,
int stderrFD, Child & child)
{
Pipe to, from;
to.create();
from.create();

child.pid = startProcess([&]() {

if (dup2(to.readSide, STDIN_FILENO) == -1)
throw SysError("cannot dup input pipe to stdin");

if (dup2(from.writeSide, STDOUT_FILENO) == -1)
throw SysError("cannot dup output pipe to stdout");

if (dup2(stderrFD, STDERR_FILENO) == -1)
throw SysError("cannot dup stderr");

Strings argv({"ssh", "-x", "-a", sshName, "--", "nix-store", "--serve", "--write"});

execvp("ssh", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast

throw SysError("cannot start ssh");
});

to.readSide.close();
from.writeSide.close();

child.to = to.writeSide.borrow();
child.from = from.readSide.borrow();
}

static void copyClosureTo(std::shared_ptr<StoreAPI> store,
FdSource & from, FdSink & to, const PathSet & paths,
bool useSubstitutes = false)
{
PathSet closure;
for (auto & path : paths)
computeFSClosure(*store, path, closure);

Paths sorted = topoSortPaths(*store, closure);

/* Send the "query valid paths" command with the "lock" option
enabled. This prevents a race where the remote host
garbage-collect paths that are already there. Optionally, ask
the remote host to substitute missing paths. */
writeInt(cmdQueryValidPaths, to);
writeInt(1, to); // == lock paths
writeInt(useSubstitutes, to);
writeStrings(sorted, to);
to.flush();

/* Get back the set of paths that are already valid on the remote
host. */
auto present = readStorePaths<PathSet>(from);

PathSet missing;
std::set_difference(closure.begin(), closure.end(), present.begin(), present.end(),
std::inserter(missing, missing.end()));

printMsg(lvlError, format("sending %1% missing paths") % missing.size());
if (missing.empty()) return;

throw Error("NOT IMPL 1");
}

static void copyClosureFrom(std::shared_ptr<StoreAPI> store,
FdSource & from, FdSink & to, const PathSet & paths)
{
writeInt(cmdExportPaths, to);
writeInt(0, to); // == don't sign
writeStrings(paths, to);
to.flush();
store->importPaths(false, from);
}

void buildRemote(std::shared_ptr<StoreAPI> store,
const string & sshName, const string & sshKey,
const Path & drvPath, const Derivation & drv,
const nix::Path & logDir, RemoteResult & result)
{
string base = baseNameOf(drvPath);
Path logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);

createDirs(dirOf(logFile));

AutoCloseFD logFD(open(logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666));
if (logFD == -1) throw SysError(format("creating log file ‘%1%’") % logFile);

Child child;
openConnection(sshName, sshKey, logFD, child);

logFD.close();

FdSource from(child.from);
FdSink to(child.to);

/* Handshake. */
writeInt(SERVE_MAGIC_1, to);
writeInt(SERVE_PROTOCOL_VERSION, to);
to.flush();

unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_2)
throw Error(format("protocol mismatch with ‘nix-store --serve’ on ‘%1%’") % sshName);
unsigned int version = readInt(from);
if (GET_PROTOCOL_MAJOR(version) != 0x200)
throw Error(format("unsupported ‘nix-store --serve’ protocol version on ‘%1%’") % sshName);

/* Copy the input closure. */
printMsg(lvlError, format("sending closure of ‘%1%’ to ‘%2%’") % drvPath % sshName);
copyClosureTo(store, from, to, PathSet({drvPath}));

/* Do the build. */
printMsg(lvlError, format("building ‘%1%’ on ‘%2%’") % drvPath % sshName);
writeInt(cmdBuildPaths, to);
writeStrings(PathSet({drvPath}), to);
writeInt(3600, to); // == maxSilentTime, FIXME
writeInt(7200, to); // == buildTimeout, FIXME
to.flush();
result.startTime = time(0);
int res = readInt(from);
result.stopTime = time(0);
if (res) {
result.errorMsg = (format("%1% on ‘%2%’") % readString(from) % sshName).str();
if (res == 100) result.status = RemoteResult::rrPermanentFailure;
else if (res == 101) result.status = RemoteResult::rrTimedOut;
else result.status = RemoteResult::rrMiscFailure;
return;
}

/* Copy the output paths. */
printMsg(lvlError, format("copying outputs of ‘%1%’ from ‘%2%’") % drvPath % sshName);
PathSet outputs;
for (auto & output : drv.outputs)
outputs.insert(output.second.path);
copyClosureFrom(store, from, to, outputs);

/* Shut down the connection. */
child.to.close();
child.pid.wait(true);

result.status = RemoteResult::rrSuccess;
}
file addition: build-remote.hh (----------)

[4.187]

#pragma once

#include "store-api.hh"
#include "derivations.hh"

struct RemoteResult
{
enum {
rrSuccess = 0,
rrPermanentFailure = 1,
rrTimedOut = 2,
rrMiscFailure = 3
} status = rrMiscFailure;
std::string errorMsg;
time_t startTime = 0, stopTime = 0;
};

void buildRemote(std::shared_ptr<nix::StoreAPI> store,
const std::string & sshName, const std::string & sshKey,
const nix::Path & drvPath, const nix::Derivation & drv,
const nix::Path & logDir, RemoteResult & result);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 11

[4.4943]

[4.0]

#include "build-remote.hh"
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 137

[4.657]

[4.5993]

std::atomic_bool destroyed;

Step() : destroyed(false) { }
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 145

[4.745]

[4.745]

}
};

struct Machine
{
typedef std::shared_ptr<Machine> ptr;

std::string sshName, sshKey;
std::set<std::string> systemTypes, supportedFeatures, mandatoryFeatures;
unsigned int maxJobs = 1;
float speedFactor = 1.0;

Sync<unsigned int> currentJobs;

Machine()
{
auto currentJobs_(currentJobs.lock());
*currentJobs_ = 0;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 168

[4.6102]

[4.6102]

/* A RAII helper that manages the currentJobs field of Machine
objects. */
struct MachineReservation
{
typedef std::shared_ptr<MachineReservation> ptr;
Machine::ptr machine;
MachineReservation(Machine::ptr machine) : machine(machine)
{
auto currentJobs_(machine->currentJobs.lock());
(*currentJobs_)++;
}
~MachineReservation()
{
auto currentJobs_(machine->currentJobs.lock());
if (*currentJobs_ > 0) (*currentJobs_)--;
}
};
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 191

[4.911]→[4.911:947](∅→∅)

std::thread queueMonitorThread;

[4.911]

[3.20]

Path hydraData, logDir;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 214

[3.96]

[3.96]

/* CV for waking up the dispatcher. */
std::condition_variable dispatcherWakeup;
std::mutex dispatcherMutex;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 221

[4.1020]

[4.6420]

/* The build machines. */
typedef std::list<Machine::ptr> Machines;
Sync<Machines> machines;

/* The currently active builder threads. FIXME: We could re-use
these, but since they're fairly long-running, it's probably not
worth it. */
// std::vector<std::thread> builderThreads;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 235

[4.6457]

[3.164]

void loadMachines();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 242

[4.6731]→[4.6731:6820](∅→∅)

void finishBuildStep(pqxx::work & txn, time_t stopTime, BuildID buildId, int stepNr,

[4.6731]

[4.6820]

void finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime, BuildID buildId, int stepNr,
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 261

[4.7289]→[4.1265:1304](∅→∅)

void builderThreadEntry(int slot);

[4.7289]

[4.1304]

/* The thread that selects and starts runnable builds. */
void dispatcher();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 264

[4.1305]→[4.1305:1376](∅→∅)

void doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step);

[4.1305]

[4.7327]

void wakeDispatcher();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 266

[4.7328]

[4.7328]

MachineReservation::ptr findMachine(Step::ptr step);

void builder(Step::ptr step, MachineReservation::ptr reservation);

void doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
Machine::ptr machine);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 282

[4.7503]

[4.7503]

hydraData = getEnv("HYDRA_DATA");
if (hydraData == "") throw Error("$HYDRA_DATA must be set");

logDir = canonPath(hydraData + "/build-logs");
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 298

[4.7731]

[4.7731]

void State::loadMachines()
{
Path machinesFile = getEnv("NIX_REMOTE_SYSTEMS", "/etc/nix/machines");
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 304

[4.7732]

[4.7732]

Machines newMachines;

if (pathExists(machinesFile)) {

for (auto line : tokenizeString<Strings>(readFile(machinesFile), "\n")) {
line = trim(string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue;
tokens.resize(7);

auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2];
if (tokens[3] != "")
string2Int(tokens[3], machine->maxJobs);
else
machine->maxJobs = 1;
machine->speedFactor = atof(tokens[4].c_str());
machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
newMachines.push_back(machine);
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 328

[4.7733]

[3.331]

} else {
auto machine = std::make_shared<Machine>();
machine->sshName = "localhost";
machine->systemTypes = StringSet({settings.thisSystem});
if (settings.thisSystem == "x86_64-linux")
machine->systemTypes.insert("i686-linux");
machine->maxJobs = settings.maxBuildJobs;
newMachines.push_back(machine);
}

auto machines_(machines.lock());
*machines_ = newMachines;
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 378

[4.9262]→[4.9262:9354](∅→∅)

void State::finishBuildStep(pqxx::work & txn, time_t stopTime, BuildID buildId, int stepNr,

[4.9262]

[4.9354]

void State::finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime, BuildID buildId, int stepNr,
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 381

[4.9438]

[4.391]

assert(startTime);
assert(stopTime);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 384

[4.413]→[4.9471:9611](∅→∅),[4.9471]→[4.9471:9611](∅→∅)

("update BuildSteps set busy = 0, status = $1, propagatedFrom = $4, errorMsg = $5, stopTime = $6 where build = $2 and stepnr = $3")

[4.413]

[4.414]

("update BuildSteps set busy = 0, status = $1, propagatedFrom = $4, errorMsg = $5, startTime = $6, stopTime = $7 where build = $2 and stepnr = $3")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 388

[4.535]→[4.535:577](∅→∅)

(stopTime, stopTime != 0).exec();

[4.535]

[4.9821]

(startTime)(stopTime).exec();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 464

[4.10582]

[3.606]

printMsg(lvlInfo, format("aborting GC'ed build %1%") % build->id);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 484

[4.2064]

[4.11213]

printMsg(lvlInfo, format("cached build %1%") % build->id);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 506

[4.3558]

[4.3558]

printMsg(lvlInfo, format("added build %1% (top-level step %2%, %3% new runnable steps)")
% build->id % step->drvPath % newRunnable.size());
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 586

[4.12643]

[4.5073]

if (step->destroyed) return;
step->destroyed = true;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 674

[4.14048]

[4.6558]

printMsg(lvlInfo, format("step ‘%1%’ is now runnable") % step->drvPath);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 686

[4.2459]→[4.396:429](∅→∅)

runnableWakeup.notify_one();

[4.2459]

[4.14262]

wakeDispatcher();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 690

[4.14266]→[4.2489:2530](∅→∅)

void State::builderThreadEntry(int slot)

[4.14266]

[4.2530]

void State::dispatcher()
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 692

[4.2532]→[4.2532:2577](∅→∅)

auto store = openStore(); // FIXME: pool

[4.2532]

[4.2577]

while (!exitRequested) {
printMsg(lvlError, "dispatcher woken up");
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 695

[4.2578]→[4.2578:2598](∅→∅),[4.2598]→[4.6726:6793](∅→∅),[4.6793]→[4.2598:2622](∅→∅),[4.2598]→[4.2598:2622](∅→∅)

while (true) {
/* Sleep until a runnable build step becomes available. */
Step::ptr step;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 697

[4.6839]→[4.430:573](∅→∅),[4.573]→[4.6922:7068](∅→∅),[4.6922]→[4.6922:7068](∅→∅)

while (runnable_->empty() && !exitRequested)
runnable_.wait(runnableWakeup);
if (exitRequested) break;
auto weak = *runnable_->begin();
runnable_->pop_front();
step = weak.lock();
if (!step) continue;

[4.6839]

[4.2842]

printMsg(lvlError, format("%1% runnable builds") % runnable_->size());

/* FIXME: we're holding the runnable lock too long
here. This could be more efficient. */

for (auto i = runnable_->begin(); i != runnable_->end(); ) {
auto step = i->lock();

/* Delete dead steps. */
if (!step) {
i = runnable_->erase(i);
continue;
}

auto reservation = findMachine(step);
if (!reservation) {
printMsg(lvlError, format("cannot execute step ‘%1%’ right now") % step->drvPath);
++i;
continue;
}

printMsg(lvlInfo, format("WOOHOO: starting step ‘%1%’ on machine ‘%2%’")
% step->drvPath % reservation->machine->sshName);
i = runnable_->erase(i);

auto builderThread = std::thread(&State::builder, this, step, reservation);
builderThread.detach(); // FIXME?
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 727

[4.2853]→[4.7069:7093](∅→∅),[4.7093]→[4.2853:2950](∅→∅),[4.2853]→[4.2853:2950](∅→∅),[4.2951]→[4.2951:2985](∅→∅)

/* Build it. */
printMsg(lvlError, format("slot %1%: got build step ‘%2%’") % slot % step->drvPath);
doBuildStep(store, step);

[4.2853]

[4.2985]

/* Sleep until we're woken up (either because a runnable build
is added, or because a build finishes). */
{
std::unique_lock<std::mutex> lock(dispatcherMutex);
dispatcherWakeup.wait(lock);
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 735

[4.2992]→[4.2992:3040](∅→∅)

printMsg(lvlError, "builder thread exits");

[4.2992]

[4.3040]

printMsg(lvlError, "dispatcher exits");
}

void State::wakeDispatcher()
{
{ std::lock_guard<std::mutex> lock(dispatcherMutex); } // barrier
dispatcherWakeup.notify_all();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 746

[4.3044]→[4.3044:3117](∅→∅)

void State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step)

[4.3044]

[4.14306]

MachineReservation::ptr State::findMachine(Step::ptr step)
{
auto machines_(machines.lock());

for (auto & machine : *machines_) {
if (!has(machine->systemTypes, step->drv.platform)) continue;
// FIXME: check features
{
auto currentJobs_(machine->currentJobs.lock());
if (*currentJobs_ >= machine->maxJobs) continue;
}
return std::make_shared<MachineReservation>(machine);
}

/* FIXME: distinguish between permanent failures (a matching
machine doesn't exist) and temporary failures (a matching
machine is not available). */

return 0;
}

void State::builder(Step::ptr step, MachineReservation::ptr reservation)
{
try {
auto store = openStore(); // FIXME: pool
doBuildStep(store, step, reservation->machine);
} catch (std::exception & e) {
printMsg(lvlError, format("build thread for ‘%1%’: %2%") % step->drvPath % e.what());
// FIXME: put step back in runnable and retry
}

/* Release the machine and wake up the dispatcher. */
assert(reservation.unique());
reservation = 0;
wakeDispatcher();

printMsg(lvlError, "builder exits");
}

void State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
Machine::ptr machine)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 825

[3.707]→[4.15518:15550](∅→∅),[4.15518]→[4.15518:15550](∅→∅)

time_t startTime = time(0);

[3.707]

[4.15550]

RemoteResult result;
result.startTime = time(0);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 830

[3.739]→[4.15602:15674](∅→∅),[4.15602]→[4.15602:15674](∅→∅)

stepNr = createBuildStep(txn, startTime, build, step, bssBusy);

[3.739]

[4.15674]

stepNr = createBuildStep(txn, result.startTime, build, step, bssBusy);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 834

[4.15703]→[4.15703:15755](∅→∅)

bool success = false;
std::string errorMsg;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 835

[4.15765]→[4.15765:15842](∅→∅)

store->buildPaths(PathSet({step->drvPath}));
success = true;

[4.15765]

[4.15842]

buildRemote(store, machine->sshName, machine->sshKey, step->drvPath, step->drv, logDir, result);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 837

[4.15868]→[4.15868:15896](∅→∅)

errorMsg = e.msg();

[4.15868]

[4.15896]

result.status = RemoteResult::rrMiscFailure;
result.errorMsg = e.msg();
printMsg(lvlError, format("ERROR: %1%") % e.msg());
abort();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 843

[4.15903]→[4.15903:15934](∅→∅)

time_t stopTime = time(0);

[4.15903]

[4.15934]

if (!result.stopTime) result.stopTime = time(0);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 846

[4.15956]→[4.3118:3175](∅→∅)

if (success) res = getBuildResult(store, step->drv);

[4.15956]

[4.16006]

if (result.status == RemoteResult::rrSuccess) res = getBuildResult(store, step->drv);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 877

[4.16153]→[4.16153:16176](∅→∅)

if (success) {

[4.16153]

[4.16176]

if (result.status == RemoteResult::rrSuccess) {
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 879

[4.16177]→[4.16177:16252](∅→∅)

finishBuildStep(txn, stopTime, build->id, stepNr, bssSuccess);

[4.16177]

[4.16252]

finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, bssSuccess);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 884

[4.9047]→[4.16491:16573](∅→∅),[4.16491]→[4.16491:16573](∅→∅)

markSucceededBuild(txn, build2, res, false, startTime, stopTime);

[4.9047]

[4.16587]

markSucceededBuild(txn, build2, res, false, result.startTime, result.stopTime);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 889

[4.16702]→[4.16702:16786](∅→∅)

finishBuildStep(txn, stopTime, build->id, stepNr, bssFailed, errorMsg);

[4.16702]

[4.16786]

finishBuildStep(txn, result.startTime, result.stopTime, build->id, stepNr, bssFailed, result.errorMsg);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 893

[4.16875]→[4.16875:16969](∅→∅)

createBuildStep(txn, stopTime, build2, step, bssFailed, errorMsg, build->id);

[4.16875]

[4.16969]

createBuildStep(txn, result.stopTime, build2, step, bssFailed, result.errorMsg, build->id);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 898

[4.9139]

[4.17101]

printMsg(lvlError, format("marking build %1% as failed") % build2->id);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 903

[4.17393]→[4.17393:17464](∅→∅)

(startTime)
(stopTime).exec();

[4.17393]

[4.17464]

(result.startTime)
(result.stopTime).exec();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 917

[4.9426]→[4.9426:9478](∅→∅)

if (build2->toplevel == step || !success) {

[4.9426]

[4.9478]

if (build2->toplevel == step || result.status != RemoteResult::rrSuccess) {
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 925

[4.9718]→[4.17637:17669](∅→∅),[4.17637]→[4.17637:17669](∅→∅)

destroyStep(step, success);

[4.9718]

[4.17669]

destroyStep(step, result.status == RemoteResult::rrSuccess);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 932

[4.17826]

[4.578]

printMsg(lvlError, format("marking build %1% as succeeded") % build->id);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 969

[4.3286]→[4.574:640](∅→∅)

queueMonitorThread = std::thread(&State::queueMonitor, this);

[4.3286]

[4.3363]

loadMachines();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 971

[4.3364]→[4.641:686](∅→∅),[4.686]→[4.3379:3411](∅→∅),[4.3379]→[4.3379:3411](∅→∅),[4.3411]→[4.687:771](∅→∅)

std::vector<std::thread> builderThreads;
for (int n = 0; n < 4; n++)
builderThreads.push_back(std::thread(&State::builderThreadEntry, this, n));

[4.3364]

[4.771]

auto queueMonitorThread = std::thread(&State::queueMonitor, this);

auto dispatcherThread = std::thread(&State::dispatcher, this);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 989

[4.1131]→[4.1131:1272](∅→∅)

{ auto runnable_(runnable.lock()); } // barrier
runnableWakeup.notify_all();
for (auto & thread : builderThreads) thread.join();

[4.1131]

[3.811]

wakeDispatcher();
dispatcherThread.join();
replacement in src/lib/Hydra/Helper/Nix.pm at line 136

[2.132]→[2.132:251](∅→∅)

for ($fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
return $_ if (-f $_);

[2.132]

[2.251]

my $fn2 = Hydra::Model::DB::getHydraPath . "/build-logs/";
for ($fn2 . $bucketed, $fn . $bucketed . ".bz2", $fn . $bucketed, $fn . $base . ".bz2", $fn . $base) {
return $_ if -f $_;