hydra-eval-jobs: Ugly hackery to reduce memory usage

[?]

Jun 5, 2018, 10:18 AM

5MP35ORVVILU3GXYITYXD754QFZCOZ3X45QO54K2GW5UKNHX66SAC

Dependencies

[2] 4GTOWRFW hydra-eval-jobs: Fix build
[3] HP5WJLQU hydra-eval-{jobs,jobset}: Pass file name as <...>
[4] ZVSHXE3K Allow setting GC_INITIAL_HEAP_SIZE for hydra-eval-jobs
[5] FGLKIVYS Fix build against Nix master
[6] AEKIREIH * Updated hydra_eval_jobs for the new evaluator.
[7] YNGIYQRF hydra-eval-jobs: Don't keep track of used inputs
[8] 4N5APGRG * Start of a helper tool to evaluate job expressions efficiently.
[9] WMWBCZPR
[10] VTMBJVIS hydra-eval-jobs: Disable the build hook.
[11] 52JSLNC6 * Build fix.
[12] MIC2O6ZF Use evalFile() instead of parseExprFromFile()
[13] M5BEPXTE Tweak debug output
[14] I2HYJBML Debug tweak
[15] FUB37KFU Use Nix's restricted evaluation mode
[16] STZE4KKR Fix build against Nix master
[17] DKJFD6JN Process Nix API changes
[18] FGUL3HAZ hydra-eval-jobs: Don't go into an infinite recursion
[19] IMQRX4MP hydra-eval-jobs: Use JSON instead of XML
[20] 7YCFGMZB * Report evaluation errors per job, and don't bail out if a job fails
[21] HNGPGVBT Fix build
[22] 77VF5TC6 * For ease of use during debugging, don't require --gc-roots-dir.
[23] POPU2ATH * hydra_scheduler: use eval-jobs.
[24] 4LWGZL33
[25] 7GKAIP3V Fix build and handling of string inputs starting with a dash
[26] VTNP5KDO * Don't catch all errors. Some errors mess up the connection with the
[27] 3PNG7NIB Remove trailing whitespace
[*] KD5237CU * eval-jobs now efficiently evaluates all Hydra jobs from a
[*] L4LBF7UF Handle derivations without a system attribute
[*] EBG4P4SY Doh - clear $NIX_PATH properly

Change contents

edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 18

[6.452]

[6.452]

#include <sys/types.h>
#include <sys/wait.h>
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 55

[29.1407]

[29.1407]

static std::string lastAttrPath;
static bool comma = false;
static size_t maxHeapSize;

struct BailOut { };
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 65

[6.1574]

[6.369]

bool lte(const std::string & s1, const std::string & s2)
{
size_t p1 = 0, p2 = 0;

while (true) {
if (p1 == s1.size()) return p2 == s2.size();
if (p2 == s2.size()) return true;

auto d1 = s1.find('.', p1);
auto d2 = s2.find('.', p2);

auto c = s1.compare(p1, d1 - p1, s2, p2, d2 - p2);

if (c < 0) return true;
if (c > 0) return false;

p1 = d1 == std::string::npos ? s1.size() : d1 + 1;
p2 = d2 == std::string::npos ? s2.size() : d2 + 1;
}
}
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 90

[6.257]

[29.1529]

if (lastAttrPath != "" && lte(attrPath, lastAttrPath)) return;
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 110

[30.123]

[30.123]

if (comma) { std::cout << ","; comma = false; }
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 157

[6.16]

[6.16]

}

GC_prof_stats_s gc;
GC_get_prof_stats(&gc, sizeof(gc));

if (gc.heapsize_full > maxHeapSize) {
printInfo("restarting hydra-eval-jobs after job '%s' because heap size is at %d bytes", attrPath, gc.heapsize_full);
lastAttrPath = attrPath;
throw BailOut();
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 171

[6.120]→[6.502:544](∅→∅),[6.544]→[6.219:280](∅→∅),[6.280]→[6.605:690](∅→∅),[6.605]→[6.605:690](∅→∅)

for (auto & i : *v.attrs)
findJobs(state, top, autoArgs, *i.value,
(attrPath.empty() ? "" : attrPath + ".") + (string) i.name);

[6.120]

[6.336]

for (auto & i : v.attrs->lexicographicOrder()) {
std::string name(i->name);

/* Skip jobs with dots in the name. */
if (name.find('.') != std::string::npos) {
printError("skipping job with illegal name '%s'", name);
continue;
}

findJobs(state, top, autoArgs, *i->value,
(attrPath.empty() ? "" : attrPath + ".") + name);
}
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 202

[6.30]→[6.0:10](∅→∅)

{

[6.30]

[2.411]

if (comma) { std::cout << ","; comma = false; }
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 205

[6.1824]→[6.11:21](∅→∅)

}
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 211

[6.570]

[31.0]

assert(lte("abc", "def"));
assert(lte("abc", "def.foo"));
assert(!lte("def", "abc"));
assert(lte("nixpkgs.hello", "nixpkgs"));
assert(lte("nixpkgs.hello", "nixpkgs.hellooo"));
assert(lte("gitAndTools.git-annex.x86_64-darwin", "gitAndTools.git-annex.x86_64-linux"));
assert(lte("gitAndTools.git-annex.x86_64-linux", "gitAndTools.git-annex-remote-b2.aarch64-linux"));
edit in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 231

[4.336]

[6.80]

maxHeapSize = config->getIntOption("evaluator_max_heap_size", 1UL << 30);
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 257

[6.1]→[6.0:119](∅→∅),[6.119]→[6.722:754](∅→∅)

/* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
settings.builders = "";

[6.1]

[6.158]

JSONObject json(std::cout, true);
std::cout.flush();
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 260

[6.159]→[6.1:105](∅→∅),[6.1]→[6.1:105](∅→∅),[6.105]→[5.0:42](∅→∅)

/* Prevent access to paths outside of the Nix search path and
to the environment. */
evalSettings.restrictEval = true;

[6.159]

[6.854]

do {

Pipe pipe;
pipe.create();
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 265

[6.855]→[6.1196:1272](∅→∅)

if (releaseExpr == "") throw UsageError("no expression specified");

[6.855]

[6.808]

ProcessOptions options;
options.allowVfork = false;
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 268

[6.809]→[6.1273:1366](∅→∅)

if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");

[6.809]

[6.810]

auto pid = startProcess([&]() {
pipe.readSide = -1;
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 271

[6.811]→[6.794:851](∅→∅)

EvalState state(myArgs.searchPath, openStore());

[6.811]

[6.880]

if (lastAttrPath != "") debug("resuming from '%s'", lastAttrPath);
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 273

[6.881]→[6.852:910](∅→∅)

Bindings & autoArgs = *myArgs.getAutoArgs(state);

[6.881]

[6.1624]

/* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
settings.builders = "";
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 276

[6.1]→[6.1927:1944](∅→∅),[6.1944]→[3.37:99](∅→∅)

Value v;
state.evalFile(lookupFileArg(state, releaseExpr), v);

[6.1625]

[6.1021]

/* Prevent access to paths outside of the Nix search path and
to the environment. */
evalSettings.restrictEval = true;
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 280

[6.1022]→[2.453:495](∅→∅),[2.495]→[6.1886:1934](∅→∅),[6.1886]→[6.1886:1934](∅→∅)

JSONObject json(std::cout, true);
findJobs(state, json, autoArgs, v, "");

[6.1022]

[6.1022]

if (releaseExpr == "") throw UsageError("no expression specified");
replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 282

[6.1023]→[6.2127:2155](∅→∅)

state.printStats();

[6.1023]

[6.2155]

if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");

EvalState state(myArgs.searchPath, openStore());

Bindings & autoArgs = *myArgs.getAutoArgs(state);

Value v;
state.evalFile(lookupFileArg(state, releaseExpr), v);

comma = lastAttrPath != "";

try {
findJobs(state, json, autoArgs, v, "");
lastAttrPath = "";
} catch (BailOut &) { }

writeFull(pipe.writeSide.get(), lastAttrPath);

exit(0);
}, options);

pipe.writeSide = -1;

int status;
while (true) {
checkInterrupt();
if (waitpid(pid, &status, 0) == pid) break;
if (errno != EINTR) continue;
}

if (status != 0)
throw Exit(WIFEXITED(status) ? WEXITSTATUS(status) : 99);

lastAttrPath = drainFD(pipe.readSide.get());
} while (lastAttrPath != "");