Keep stats for the Hydra auto scaler

Dependencies

• [2] IK2UBDAU Revive jobset scheduling
• [3] MHVIT4JY Split hydra-queue-runner.cc more
• [4] OG3Z3QGC Namespace cleanup
• [5] YR2IM6Y5 Temporarily disable machines after a connection failure
• [6] 4I2HF4L3 Unindent
• [7] HJOEIMLR Refactor
• [8] A2GL5FOZ Moar stats
• [9] 7LB6QBXY Keep track of the number of build steps that are being built
• [10] 46ADBTMQ Start steps in order of ascending build ID
• [*] 24BMQDZA Start of single-process hydra-queue-runner
• [*] SODOV2CM Automatically reload $NIX_REMOTE_SYSTEMS when it changes
• [*] RQUAATWB Add status dump facility
• [*] CNLNT3T4 Allow only 1 thread to send a closure to a given machine at the same time

Change contents

• replacement in src/hydra-queue-runner/builder.cc at line 9
  [3.223]→[3.223:325](∅→∅)
  − void State::builder(Step::ptr step, Machine::ptr machine, std::shared_ptr<MaintainCount> reservation)
  [3.223]

  [3.325]
  + void State::builder(MachineReservation::ptr reservation)
• edit in src/hydra-queue-runner/builder.cc at line 14
  [3.388]

  [3.388]
  +

  + auto step = reservation->step;
• replacement in src/hydra-queue-runner/builder.cc at line 19
  [3.448]→[3.448:499](∅→∅)
  − retry = doBuildStep(store, step, machine);
  [3.448]

  [3.499]
  + retry = doBuildStep(store, step, reservation->machine);
• replacement in src/hydra-queue-runner/builder.cc at line 22
  [3.627]→[3.627:687](∅→∅)
  − % step->drvPath % machine->sshName % e.what());
  [3.627]

  [3.687]
  + % step->drvPath % reservation->machine->sshName % e.what());
• replacement in src/hydra-queue-runner/counter.hh at line 11
  [3.399]→[3.399:429](∅→∅)
  − ~MaintainCount() { c--; }
  [3.399]

  [3.429]
  + ~MaintainCount() { auto prev = c--; assert(prev); }
• edit in src/hydra-queue-runner/dispatcher.cc at line 3
  [3.14668]

  [3.14668]
  + #include <unordered_map>
• edit in src/hydra-queue-runner/dispatcher.cc at line 146
  [3.292]

  [3.292]
  + std::unordered_map<std::string, unsigned int> runnablePerType;
• edit in src/hydra-queue-runner/dispatcher.cc at line 160
  [3.422]

  [3.18020]
  +

  + runnablePerType[step->drv.platform]++;
• edit in src/hydra-queue-runner/dispatcher.cc at line 222
  [3.1899]

  [3.4106]
  + assert(runnablePerType[step->drv.platform]);

  + runnablePerType[step->drv.platform]--;
• replacement in src/hydra-queue-runner/dispatcher.cc at line 228
  [3.4226]→[3.4226:4326](∅→∅),[3.4367]→[3.19141:19142](∅→∅),[3.19141]→[3.19141:19142](∅→∅),[3.19142]→[3.4368:4472](∅→∅)
  − auto reservation = std::make_shared<MaintainCount>(mi.machine->state->currentJobs);

  −

  − auto builderThread = std::thread(&State::builder, this, step, mi.machine, reservation);
  [3.4226]

  [3.4472]
  + auto builderThread = std::thread(&State::builder, this,

  + std::make_shared<MachineReservation>(*this, step, mi.machine));
• edit in src/hydra-queue-runner/dispatcher.cc at line 237
  [3.4615]

  [3.4615]
  + }

  +

  + /* Update the stats for the auto-scaler. */

  + {

  + auto machineTypes_(machineTypes.lock());

  +

  + for (auto & i : *machineTypes_)

  + i.second.runnable = 0;

  +

  + for (auto & i : runnablePerType)

  + (*machineTypes_)[i.first].runnable = i.second;
• edit in src/hydra-queue-runner/dispatcher.cc at line 283
  [2.1721]

  [2.1721]
  + }

  + }

  +

  +

  + State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine)

  + : state(state), step(step), machine(machine)

  + {

  + machine->state->currentJobs++;

  +

  + {

  + auto machineTypes_(state.machineTypes.lock());

  + (*machineTypes_)[step->drv.platform].running++;

  + }

  + }

  +

  +

  + State::MachineReservation::~MachineReservation()

  + {

  + auto prev = machine->state->currentJobs--;

  + assert(prev);

  + if (prev == 1)

  + machine->state->idleSince = time(0);

  +

  + {

  + auto machineTypes_(state.machineTypes.lock());

  + auto & machineType = (*machineTypes_)[step->drv.platform];

  + assert(machineType.running);

  + machineType.running--;

  + if (machineType.running == 0)

  + machineType.lastActive = time(0);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 456
  [13.4639]

  [13.4639]
  + if (s->currentJobs == 0)

  + nested2.attr("idleSince", s->idleSince);
• edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 478
  [2.2189]

  [14.401]
  + {

  + root.attr("machineTypes");

  + JSONObject nested(out);

  + auto machineTypes_(machineTypes.lock());

  + for (auto & i : *machineTypes_) {

  + nested.attr(i.first);

  + JSONObject nested2(out);

  + nested2.attr("runnable", i.second.runnable);

  + nested2.attr("running", i.second.running);

  + if (i.second.running == 0)

  + nested2.attr("lastActive", i.second.lastActive);

  + }

  + }
• edit in src/hydra-queue-runner/state.hh at line 211
  [3.3777]

  [15.158]
  + std::atomic<time_t> idleSince{0};
• edit in src/hydra-queue-runner/state.hh at line 321
  [3.6777]

  [3.6777]
  +

  + /* Statistics per machine type for the Hydra auto-scaler. */

  + struct MachineType

  + {

  + unsigned int runnable{0}, running{0};

  + time_t lastActive{0};

  + };

  +

  + Sync<std::map<std::string, MachineType>> machineTypes;

  +

  + struct MachineReservation

  + {

  + typedef std::shared_ptr<MachineReservation> ptr;

  + State & state;

  + Step::ptr step;

  + Machine::ptr machine;

  + MachineReservation(State & state, Step::ptr step, Machine::ptr machine);

  + ~MachineReservation();

  + };
• replacement in src/hydra-queue-runner/state.hh at line 392
  [3.8187]→[3.8187:8287](∅→∅)
  − void builder(Step::ptr step, Machine::ptr machine, std::shared_ptr<MaintainCount> reservation);
  [3.8187]

  [3.8287]
  + void builder(MachineReservation::ptr reservation);