Revive jobset scheduling

[?]

Aug 10, 2015, 11:30 PM

IK2UBDAU6QKUXHJG3SXJKYGIIXRDKI6UVRTFC6ZVDXDCGNCMEWVAC

Dependencies

[2] WE5Q2NVI Allow build to be bumped to the front of the queue via the web interface
[3] MHVIT4JY Split hydra-queue-runner.cc more
[4] T5BIOVJE Add support for tracking custom metrics
[5] 46ADBTMQ Start steps in order of ascending build ID
[*] 4I2HF4L3 Unindent
[*] 24BMQDZA Start of single-process hydra-queue-runner
[*] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
[*] RQUAATWB Add status dump facility
[*] HJOEIMLR Refactor
[*] NAYQT2GT hydra-queue-runner: Use cmdBuildDerivation

Change contents

edit in src/hydra-queue-runner/builder.cc at line 133

[3.4778]

[3.4778]

/* Account the time we spent building this step by dividing it
among the jobsets that depend on it. */
{
auto step_(step->state.lock());
// FIXME: loss of precision.
time_t charge = (result.stopTime - result.startTime) / step_->jobsets.size();
for (auto & jobset : step_->jobsets)
jobset->addStep(result.startTime, charge);
}
edit in src/hydra-queue-runner/dispatcher.cc at line 1

[3.14628]→[3.0:20](∅→∅)

#include <iostream>
edit in src/hydra-queue-runner/dispatcher.cc at line 56

[7.627]

[3.21]

/* Prune old historical build step info from the jobsets. */
{
auto jobsets_(jobsets.lock());
for (auto & jobset : *jobsets_) {
auto s1 = jobset.second->shareUsed();
jobset.second->pruneSteps();
auto s2 = jobset.second->shareUsed();
if (s1 != s2)
printMsg(lvlDebug, format("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%")
% jobset.first.first % jobset.first.second % s1 % s2);
}
}
edit in src/hydra-queue-runner/dispatcher.cc at line 154

[3.515]

[3.515]

for (auto & step : runnableSorted) {
auto step_(step->state.lock());
step_->lowestShareUsed = 1e9;
for (auto & jobset : step_->jobsets)
step_->lowestShareUsed = std::min(step_->lowestShareUsed, jobset->shareUsed());
}
edit in src/hydra-queue-runner/dispatcher.cc at line 168

[2.156]

[2.156]

a_->lowestShareUsed != b_->lowestShareUsed ? a_->lowestShareUsed < b_->lowestShareUsed :
edit in src/hydra-queue-runner/dispatcher.cc at line 227

[3.19999]

void Jobset::addStep(time_t startTime, time_t duration)
{
auto steps_(steps.lock());
(*steps_)[startTime] = duration;
seconds += duration;
}

void Jobset::pruneSteps()
{
time_t now = time(0);
auto steps_(steps.lock());
while (!steps_->empty()) {
auto i = steps_->begin();
if (i->first > now - schedulingWindow) break;
seconds -= i->second;
steps_->erase(i);
}
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 465

[9.2016]

[10.401]

{
root.attr("jobsets");
JSONObject nested(out);
auto jobsets_(jobsets.lock());
for (auto & jobset : *jobsets_) {
nested.attr(jobset.first.first + ":" + jobset.first.second);
JSONObject nested2(out);
nested2.attr("shareUsed"); out << jobset.second->shareUsed();
nested2.attr("seconds", jobset.second->getSeconds());
}
}
edit in src/hydra-queue-runner/queue-monitor.cc at line 52

[3.21501]→[3.21501:21502](∅→∅)
edit in src/hydra-queue-runner/queue-monitor.cc at line 86

[2.748]

[3.22781]

build->jobset = createJobset(txn, build->projectName, build->jobsetName);
edit in src/hydra-queue-runner/queue-monitor.cc at line 212

[3.27613]

[3.27613]

build->propagatePriorities();
edit in src/hydra-queue-runner/queue-monitor.cc at line 235

[3.2401]→[2.749:787](∅→∅),[2.787]→[3.28327:28328](∅→∅),[3.2765]→[3.28327:28328](∅→∅),[3.28327]→[3.28327:28328](∅→∅)

build->propagatePriorities();
edit in src/hydra-queue-runner/queue-monitor.cc at line 256

[2.1300]

[2.1300]

step_->jobsets.insert(jobset);
edit in src/hydra-queue-runner/queue-monitor.cc at line 399

[3.32923]

Jobset::ptr State::createJobset(pqxx::work & txn,
const std::string & projectName, const std::string & jobsetName)
{
auto jobsets_(jobsets.lock());

auto p = std::make_pair(projectName, jobsetName);

auto i = jobsets_->find(p);
if (i != jobsets_->end()) return i->second;

auto res = txn.parameterized
("select schedulingShares from Jobsets where project = $1 and name = $2")
(projectName)(jobsetName).exec();
if (res.empty()) throw Error("missing jobset - can't happen");

auto shares = res[0]["schedulingShares"].as<unsigned int>();
if (shares == 0) shares = 1;

auto jobset = std::make_shared<Jobset>(shares);

/* Load the build steps from the last 24 hours. */
res = txn.parameterized
("select s.startTime, s.stopTime from BuildSteps s join Builds b on build = id "
"where s.startTime is not null and s.stopTime > $1 and project = $2 and jobset = $3")
(time(0) - Jobset::schedulingWindow * 10)(projectName)(jobsetName).exec();
for (auto const & row : res) {
time_t startTime = row["startTime"].as<time_t>();
time_t stopTime = row["stopTime"].as<time_t>();
jobset->addStep(startTime, stopTime - startTime);
}

(*jobsets_)[p] = jobset;
return jobset;
}
edit in src/hydra-queue-runner/state.hh at line 61

[12.3783]

[11.1954]

class Jobset
{
public:

typedef std::shared_ptr<Jobset> ptr;
typedef std::weak_ptr<Jobset> wptr;

Jobset(unsigned int shares) : shares(shares) { }

static const time_t schedulingWindow = 24 * 60 * 60;

private:
edit in src/hydra-queue-runner/state.hh at line 76

[11.1955]

[11.1955]

std::atomic<time_t> seconds{0};
std::atomic<unsigned int> shares;
edit in src/hydra-queue-runner/state.hh at line 79

[11.1956]

[11.1956]

/* The start time and duration of the most recent build steps. */
Sync<std::map<time_t, time_t>> steps;

public:

double shareUsed()
{
return (double) seconds / shares;
}

time_t getSeconds() { return seconds; }

void addStep(time_t startTime, time_t duration);

void pruneSteps();
};
edit in src/hydra-queue-runner/state.hh at line 111

[11.2233]

[11.2233]

Jobset::ptr jobset;
edit in src/hydra-queue-runner/state.hh at line 149

[11.2904]

[11.2904]

/* Jobsets to which this step belongs. Used for determining
scheduling priority. */
std::set<Jobset::ptr> jobsets;
edit in src/hydra-queue-runner/state.hh at line 162

[2.2151]

[2.2151]

/* The lowest share used of any jobset depending on this
step. */
double lowestShareUsed;
edit in src/hydra-queue-runner/state.hh at line 249

[11.4521]

[11.4521]

/* The jobsets. */
typedef std::map<std::pair<std::string, std::string>, Jobset::ptr> Jobsets;
Sync<Jobsets> jobsets;
edit in src/hydra-queue-runner/state.hh at line 350

[11.8033]

[11.8033]

Jobset::ptr createJobset(pqxx::work & txn,
const std::string & projectName, const std::string & jobsetName);