Automatically reload $NIX_REMOTE_SYSTEMS when it changes

[?]

Jun 25, 2015, 10:24 AM

SODOV2CMWA4JMIKRQNJ6MD3U3BS2XTSLINLRAG4SFY742IIJNI5QC

Dependencies

[2] FKLICOHY Prefer cached failure over unsupported system type
[3] ODCBSLFG hydra-queue-runner: Fix segfault sorting machines by load
[4] O64P4XJS Keep per-machine stats
[5] IE2PRAQU hydra-queue-runner: Send build notifications
[6] MB3TISH2 Rate-limit the number of threads copying closures at the same time
[7] N5O7VEEO Immediately abort builds that require an unsupported system type
[8] NJJ7H64S Very basic multi-threaded queue runner
[9] YZAI5GQU Implement a database connection pool
[10] HLSHCK3C Support requiredSystemFeatures
[11] NNOCZ4RO hydra-queue-runner: Improve dispatcher
[12] IWB3F4Z6 Fail builds with previously failed steps early
[13] OCZ4LSGG Automatically retry aborted builds
[14] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
[15] HUUZFPPK Fix race between the queue monitor and the builder threads
[16] GS4BE6TB Asynchronously compress build logs
[17] T2EIYJNG On SIGINT, shut down the builder threads
[18] FQQRJUO4 Mark builds as busy
[19] 24BMQDZA Start of single-process hydra-queue-runner
[20] ENXUSMSV Make concurrency more robust
[21] 63W4T5PU hydra-queue-runner: More stats
[22] GKZN4UV7 Make the queue monitor more robust, and better debug output
[23] WKJFPR77 hydra-queue-runner: Maintain count of active build steps
[24] 5AIYUMTB Basic remote building

Change contents

replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 183

[7.6162]→[4.0:202](∅→∅)

counter currentJobs{0};
counter nrStepsDone{0};
counter totalStepTime{0}; // total time for steps, including closure copying
counter totalStepBuildTime{0}; // total build time for steps

[7.6162]

[7.6]

struct State {
typedef std::shared_ptr<State> ptr;
counter currentJobs{0};
counter nrStepsDone{0};
counter totalStepTime{0}; // total time for steps, including closure copying
counter totalStepBuildTime{0}; // total build time for steps
};

State::ptr state;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 201

[7.190]→[7.745:751](∅→∅),[7.6293]→[7.745:751](∅→∅),[7.745]→[7.745:751](∅→∅),[7.751]→[7.6097:6102](∅→∅),[7.6097]→[7.6097:6102](∅→∅),[7.6102]→[7.6294:6549](∅→∅),[7.6549]→[7.69:101](∅→∅),[7.101]→[7.6632:6670](∅→∅),[7.6632]→[7.6632:6670](∅→∅),[7.6670]→[7.102:134](∅→∅)

}
};

/* A RAII helper that manages the currentJobs field of Machine
objects. */
struct MachineReservation
{
typedef std::shared_ptr<MachineReservation> ptr;
Machine::ptr machine;
MachineReservation(Machine::ptr machine) : machine(machine)
{
machine->currentJobs++;
}
~MachineReservation()
{
machine->currentJobs--;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 234

[7.6970]→[7.6970:7016](∅→∅)

typedef std::list<Machine::ptr> Machines;

[7.6970]

[7.7016]

typedef std::map<string, Machine::ptr> Machines;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 236

[7.7045]

[6.987]

Path machinesFile;
struct stat machinesFileStat;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 279

[7.901]→[7.901:926](∅→∅)

void loadMachines();

[7.901]

[7.6539]

/* (Re)load /etc/nix/machines. */
void loadMachinesFile();

/* Thread to reload /etc/nix/machines periodically. */
void monitorMachinesFile();
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 314

[7.7562]→[7.7562:7633](∅→∅)

void builder(Step::ptr step, MachineReservation::ptr reservation);

[7.7328]

[7.7633]

void builder(Step::ptr step, Machine::ptr machine, std::shared_ptr<MaintainCount> reservation);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 355

[7.7892]

[7.7503]

machinesFile = getEnv("NIX_REMOTE_SYSTEMS", "/etc/nix/machines");
machinesFileStat.st_ino = 0;
machinesFileStat.st_mtime = 0;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 362

[7.7895]→[7.7895:7922](∅→∅)

void State::loadMachines()

[7.7507]

[7.7922]

void State::loadMachinesFile()
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 364

[7.7924]→[7.7924:7999](∅→∅),[7.7999]→[7.7731:7732](∅→∅),[7.7731]→[7.7731:7732](∅→∅),[7.7732]→[7.8000:8027](∅→∅)

Path machinesFile = getEnv("NIX_REMOTE_SYSTEMS", "/etc/nix/machines");

Machines newMachines;

[7.7924]

[7.8027]

string contents;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 366

[7.8063]

[7.8063]

struct stat st;
if (stat(machinesFile.c_str(), &st) != 0)
throw SysError(format("getting stats about ‘%1%’") % machinesFile);
if (st.st_ino == machinesFileStat.st_ino && st.st_mtime == machinesFileStat.st_mtime)
return;
printMsg(lvlDebug, "reloading machines");
contents = readFile(machinesFile);
machinesFileStat = st;
} else {
StringSet systems = StringSet({settings.thisSystem});
if (settings.thisSystem == "x86_64-linux")
systems.insert("i686-linux");
contents = "localhost " + concatStringsSep(",", systems)
+ " - " + int2String(settings.maxBuildJobs) + " 1";
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 382

[7.8064]→[7.8064:8353](∅→∅)

for (auto line : tokenizeString<Strings>(readFile(machinesFile), "\n")) {
line = trim(string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue;
tokens.resize(7);

[7.8064]

[7.8353]

Machines newMachines, oldMachines;
{
auto machines_(machines.lock());
oldMachines = *machines_;
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 388

[7.8354]→[7.8354:8944](∅→∅),[7.8944]→[7.339:449](∅→∅),[7.449]→[7.8944:8998](∅→∅),[7.8944]→[7.8944:8998](∅→∅)

auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2];
if (tokens[3] != "")
string2Int(tokens[3], machine->maxJobs);
else
machine->maxJobs = 1;
machine->speedFactor = atof(tokens[4].c_str());
machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
for (auto & f : machine->mandatoryFeatures)
machine->supportedFeatures.insert(f);
newMachines.push_back(machine);
}

[7.8354]

[7.7732]

for (auto line : tokenizeString<Strings>(contents, "\n")) {
line = trim(string(line, 0, line.find('#')));
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.size() < 3) continue;
tokens.resize(7);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 394

[7.7733]→[7.8999:9012](∅→∅)

} else {
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 395

[7.9064]→[7.9064:9365](∅→∅)

machine->sshName = "localhost";
machine->systemTypes = StringSet({settings.thisSystem});
if (settings.thisSystem == "x86_64-linux")
machine->systemTypes.insert("i686-linux");
machine->maxJobs = settings.maxBuildJobs;
newMachines.push_back(machine);

[7.9064]

[7.9365]

machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2];
if (tokens[3] != "")
string2Int(tokens[3], machine->maxJobs);
else
machine->maxJobs = 1;
machine->speedFactor = atof(tokens[4].c_str());
machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
for (auto & f : machine->mandatoryFeatures)
machine->supportedFeatures.insert(f);

/* Re-use the State object of the previous machine with the
same name. */
auto i = oldMachines.find(machine->sshName);
if (i == oldMachines.end())
printMsg(lvlChatty, format("adding new machine ‘%1%’") % machine->sshName);
else
printMsg(lvlChatty, format("updating machine ‘%1%’") % machine->sshName);
machine->state = i == oldMachines.end()
? std::make_shared<Machine::State>()
: i->second->state;
newMachines[machine->sshName] = machine;
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 420

[7.9371]

[7.9371]

for (auto & m : oldMachines)
if (newMachines.find(m.first) == newMachines.end())
printMsg(lvlInfo, format("removing machine ‘%1%’") % m.first);
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 427

[7.9439]

[7.9439]

}

void State::monitorMachinesFile()
{
while (true) {
try {
// FIXME: use inotify.
sleep(60);
loadMachinesFile();
} catch (std::exception & e) {
printMsg(lvlError, format("reloading machines file: %1%") % e.what());
}
}
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 658

[2.230]→[2.230:307](∅→∅)

if (m->supportsStep(r)) { supported = true; break; }

[2.230]

[2.307]

if (m.second->supportsStep(r)) { supported = true; break; }
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 935

[3.442]→[3.442:509](∅→∅)

machinesSorted.push_back({m, m->currentJobs});

[3.442]

[7.466]

machinesSorted.push_back({m.second, m.second->state->currentJobs});
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 968

[7.1882]→[3.972:1050](∅→∅)

if (mi.machine->currentJobs >= mi.machine->maxJobs) continue;

[7.1882]

[7.1954]

if (mi.machine->state->currentJobs >= mi.machine->maxJobs) continue;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1005

[7.3244]→[3.1111:1200](∅→∅)

auto reservation = std::make_shared<MachineReservation>(mi.machine);

[7.3244]

[7.3330]

auto reservation = std::make_shared<MaintainCount>(mi.machine->state->currentJobs);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1008

[7.3376]→[7.3376:3472](∅→∅)

auto builderThread = std::thread(&State::builder, this, step, reservation);

[7.3376]

[7.3472]

auto builderThread = std::thread(&State::builder, this, step, mi.machine, reservation);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1042

[7.12578]→[7.12578:12651](∅→∅)

void State::builder(Step::ptr step, MachineReservation::ptr reservation)

[7.12578]

[7.12651]

void State::builder(Step::ptr step, Machine::ptr machine, std::shared_ptr<MaintainCount> reservation)
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1050

[7.12712]→[7.2750:2814](∅→∅)

retry = doBuildStep(store, step, reservation->machine);

[7.12712]

[7.12768]

retry = doBuildStep(store, step, machine);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1053

[7.2908]→[7.2908:2981](∅→∅)

% step->drvPath % reservation->machine->sshName % e.what());

[7.2908]

[7.12955]

% step->drvPath % machine->sshName % e.what());
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1398

[7.526]→[4.273:432](∅→∅)

machine->nrStepsDone++;
machine->totalStepTime += stepStopTime - stepStartTime;
machine->totalStepBuildTime += result.stopTime - result.startTime;

[7.526]

[7.526]

machine->state->nrStepsDone++;
machine->state->totalStepTime += stepStopTime - stepStartTime;
machine->state->totalStepBuildTime += result.stopTime - result.startTime;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1603

[7.1754]→[7.1754:1796](∅→∅)

for (auto & m : *machines_) {

[7.1754]

[7.1796]

for (auto & i : *machines_) {
auto & m(i.second);
auto & s(m->state);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1608

[7.1878]→[7.1878:1939](∅→∅),[7.1939]→[4.632:1087](∅→∅)

nested2.attr("currentJobs", m->currentJobs);
nested2.attr("nrStepsDone", m->nrStepsDone);
if (m->nrStepsDone) {
nested2.attr("totalStepTime", m->totalStepTime);
nested2.attr("totalStepBuildTime", m->totalStepBuildTime);
nested2.attr("avgStepTime"); out << (float) m->totalStepTime / m->nrStepsDone;
nested2.attr("avgStepBuildTime"); out << (float) m->totalStepBuildTime / m->nrStepsDone;

[7.1878]

[4.1087]

nested2.attr("currentJobs", s->currentJobs);
nested2.attr("nrStepsDone", s->nrStepsDone);
if (m->state->nrStepsDone) {
nested2.attr("totalStepTime", s->totalStepTime);
nested2.attr("totalStepBuildTime", s->totalStepBuildTime);
nested2.attr("avgStepTime"); out << (float) s->totalStepTime / s->nrStepsDone;
nested2.attr("avgStepBuildTime"); out << (float) s->totalStepBuildTime / s->nrStepsDone;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 1711

[7.3286]→[7.14685:14705](∅→∅)

loadMachines();

[7.3286]

[7.3363]

loadMachinesFile();
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 1713

[7.3364]

[5.2667]

std::thread(&State::monitorMachinesFile, this).detach();