Revert "hydra-eval-jobs -> nix eval-hydra-jobs"

Dependencies

• [2] 3VRZBVFX hydra-eval-jobs -> nix eval-hydra-jobs
• [3] N22GPKYT * Put info about logs / build products in the DB.
• [4] 3PNG7NIB Remove trailing whitespace
• [5] 24BMQDZA Start of single-process hydra-queue-runner
• [6] LAJHRL6O Fix tests
• [7] UKFZFDJO Remove the pretty-printed log
• [8] VU2OLHD2 Move most of AddBuilds to hydra-eval-jobset
• [9] 3HRBFWGK tests: change postgresql socket dir to /tmp
• [10] Q6VFUIC6 Remove hydra-eval-guile-jobs
• [11] 6EAXKZCU Use the overlay from the nix flake
• [12] P3M6UFMP Merge remote-tracking branch 'origin/master' into flake
• [13] UN2KZL3A Rename c -> hydra-eval-jobs
• [14] 6L3ZM55S Add font for the captcha
• [15] OB7GB5DL Test environment cleanups
• [16] 4YCF3KBG Concurrent hydra-evaluator
• [17] ZPHREC7X hydra-eval-jobset: Support flakes
• [*] GBCPEGYC Add a .gitignore
• [*] 3ZURHKX5 Update .gitignore
• [*] FV2M6MOT hydra: use autoconf/-make
• [*] T4LLYESZ * Nix expression for building Hydra.
• [*] CLXEECMF * Start putting build results in a database.
• [*] G2ZB6464 first test, not yet in buildprocess

Change contents

• edit in .gitignore at line 21
  [20.76]

  [20.99]
  + /src/hydra-eval-jobs/hydra-eval-jobs
• edit in configure.ac at line 67
  [3.56]

  [3.0]
  + src/hydra-eval-jobs/Makefile
• replacement in flake.nix at line 122
  [3.4563]→[2.0:98](∅→∅)
  − PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH
  [3.4563]

  [3.4688]
  + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
• replacement in src/Makefile.am at line 1
  [3.21]→[2.99:168](∅→∅)
  − SUBDIRS = hydra-evaluator hydra-queue-runner sql script lib root ttf
  [3.21]

  [3.62]
  + SUBDIRS = hydra-evaluator hydra-eval-jobs hydra-queue-runner sql script lib root ttf
• file addition: hydra-eval-jobs (d--r------)
  [23.4]
• file addition: Makefile.am (----------)
  [0.283]
  + bin_PROGRAMS = hydra-eval-jobs

  +

  + hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc

  + hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust

  + hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
• file addition: hydra-eval-jobs.cc (----------)
  [0.283]
  + #include <map>

  + #include <iostream>

  +

  + #define GC_LINUX_THREADS 1

  + #include <gc/gc_allocator.h>

  +

  + #include "shared.hh"

  + #include "store-api.hh"

  + #include "eval.hh"

  + #include "eval-inline.hh"

  + #include "util.hh"

  + #include "json.hh"

  + #include "get-drvs.hh"

  + #include "globals.hh"

  + #include "common-eval-args.hh"

  + #include "flake/flakeref.hh"

  + #include "flake/flake.hh"

  +

  + #include "hydra-config.hh"

  +

  + #include <sys/types.h>

  + #include <sys/wait.h>

  +

  + using namespace nix;

  +

  +

  + static Path gcRootsDir;

  +

  +

  + static void findJobs(EvalState & state, JSONObject & top,

  + Bindings & autoArgs, Value & v, const string & attrPath);

  +

  +

  + static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute)

  + {

  + Strings res;

  + std::function<void(Value & v)> rec;

  +

  + rec = [&](Value & v) {

  + state.forceValue(v);

  + if (v.type == tString)

  + res.push_back(v.string.s);

  + else if (v.isList())

  + for (unsigned int n = 0; n < v.listSize(); ++n)

  + rec(*v.listElems()[n]);

  + else if (v.type == tAttrs) {

  + auto a = v.attrs->find(state.symbols.create(subAttribute));

  + if (a != v.attrs->end())

  + res.push_back(state.forceString(*a->value));

  + }

  + };

  +

  + Value * v = drv.queryMeta(name);

  + if (v) rec(*v);

  +

  + return concatStringsSep(", ", res);

  + }

  +

  +

  + static void findJobsWrapped(EvalState & state, JSONObject & top,

  + Bindings & autoArgs, Value & vIn, const string & attrPath)

  + {

  + debug(format("at path `%1%'") % attrPath);

  +

  + checkInterrupt();

  +

  + Value v;

  + state.autoCallFunction(autoArgs, vIn, v);

  +

  + if (v.type == tAttrs) {

  +

  + auto drv = getDerivation(state, v, false);

  +

  + if (drv) {

  + Path drvPath;

  +

  + DrvInfo::Outputs outputs = drv->queryOutputs();

  +

  + if (drv->querySystem() == "unknown")

  + throw EvalError("derivation must have a ‘system’ attribute");

  +

  + {

  + auto res = top.object(attrPath);

  + res.attr("nixName", drv->queryName());

  + res.attr("system", drv->querySystem());

  + res.attr("drvPath", drvPath = drv->queryDrvPath());

  + res.attr("description", drv->queryMetaString("description"));

  + res.attr("license", queryMetaStrings(state, *drv, "license", "shortName"));

  + res.attr("homepage", drv->queryMetaString("homepage"));

  + res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email"));

  + res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100));

  + res.attr("timeout", drv->queryMetaInt("timeout", 36000));

  + res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200));

  + res.attr("isChannel", drv->queryMetaBool("isHydraChannel", false));

  +

  + /* If this is an aggregate, then get its constituents. */

  + Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate"));

  + if (a != v.attrs->end() && state.forceBool(*a->value, *a->pos)) {

  + Bindings::iterator a = v.attrs->find(state.symbols.create("constituents"));

  + if (a == v.attrs->end())

  + throw EvalError("derivation must have a ‘constituents’ attribute");

  + PathSet context;

  + state.coerceToString(*a->pos, *a->value, context, true, false);

  + PathSet drvs;

  + for (auto & i : context)

  + if (i.at(0) == '!') {

  + size_t index = i.find("!", 1);

  + drvs.insert(string(i, index + 1));

  + }

  + res.attr("constituents", concatStringsSep(" ", drvs));

  + }

  +

  + /* Register the derivation as a GC root. !!! This

  + registers roots for jobs that we may have already

  + done. */

  + auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();

  + if (gcRootsDir != "" && localStore) {

  + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));

  + if (!pathExists(root))

  + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false);

  + }

  +

  + auto res2 = res.object("outputs");

  + for (auto & j : outputs)

  + res2.attr(j.first, j.second);

  +

  + }

  + }

  +

  + else {

  + if (!state.isDerivation(v)) {

  + for (auto & i : v.attrs->lexicographicOrder()) {

  + std::string name(i->name);

  +

  + /* Skip jobs with dots in the name. */

  + if (name.find('.') != std::string::npos) {

  + printError("skipping job with illegal name '%s'", name);

  + continue;

  + }

  +

  + findJobs(state, top, autoArgs, *i->value,

  + (attrPath.empty() ? "" : attrPath + ".") + name);

  + }

  + }

  + }

  + }

  +

  + else if (v.type == tNull) {

  + // allow null values, meaning 'do nothing'

  + }

  +

  + else

  + throw TypeError(format("unsupported value: %1%") % v);

  + }

  +

  +

  + static void findJobs(EvalState & state, JSONObject & top,

  + Bindings & autoArgs, Value & v, const string & attrPath)

  + {

  + try {

  + findJobsWrapped(state, top, autoArgs, v, attrPath);

  + } catch (EvalError & e) {

  + auto res = top.object(attrPath);

  + res.attr("error", filterANSIEscapes(e.msg(), true));

  + }

  + }

  +

  +

  + int main(int argc, char * * argv)

  + {

  + /* Prevent undeclared dependencies in the evaluation via

  + $NIX_PATH. */

  + unsetenv("NIX_PATH");

  +

  + return handleExceptions(argv[0], [&]() {

  +

  + auto config = std::make_unique<::Config>();

  +

  + auto initialHeapSize = config->getStrOption("evaluator_initial_heap_size", "");

  + if (initialHeapSize != "")

  + setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1);

  +

  + initNix();

  + initGC();

  +

  + struct MyArgs : MixEvalArgs, MixCommonArgs

  + {

  + Path releaseExpr;

  + bool flake = false;

  +

  + MyArgs() : MixCommonArgs("hydra-eval-jobs")

  + {

  + mkFlag()

  + .longName("help")

  + .description("show usage information")

  + .handler([&]() {

  + printHelp(programName, std::cout);

  + throw Exit();

  + });

  +

  + mkFlag()

  + .longName("gc-roots-dir")

  + .description("garbage collector roots directory")

  + .labels({"path"})

  + .dest(&gcRootsDir);

  +

  + mkFlag()

  + .longName("dry-run")

  + .description("don't create store derivations")

  + .set(&settings.readOnlyMode, true);

  +

  + mkFlag()

  + .longName("flake")

  + .description("build a flake")

  + .set(&flake, true);

  +

  + expectArg("expr", &releaseExpr);

  + }

  + };

  +

  + MyArgs myArgs;

  + myArgs.parseCmdline(argvToStrings(argc, argv));

  +

  + JSONObject json(std::cout, true);

  + std::cout.flush();

  +

  + /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */

  + settings.builders = "";

  +

  + /* Prevent access to paths outside of the Nix search path and

  + to the environment. */

  + evalSettings.restrictEval = true;

  +

  + /* When building a flake, use pure evaluation (no access to

  + 'getEnv', 'currentSystem' etc. */

  + evalSettings.pureEval = myArgs.flake;

  +

  + if (myArgs.releaseExpr == "") throw UsageError("no expression specified");

  +

  + if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");

  +

  + EvalState state(myArgs.searchPath, openStore());

  +

  + Bindings & autoArgs = *myArgs.getAutoArgs(state);

  +

  + Value v;

  +

  + if (myArgs.flake) {

  + using namespace flake;

  +

  + auto flakeRef = parseFlakeRef(myArgs.releaseExpr);

  +

  + auto vFlake = state.allocValue();

  +

  + auto lockedFlake = lockFlake(state, flakeRef,

  + LockFlags {

  + .updateLockFile = false,

  + .useRegistries = false,

  + .allowMutable = false,

  + });

  +

  + callFlake(state, lockedFlake, *vFlake);

  +

  + auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value;

  + state.forceValue(*vOutputs);

  +

  + auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs"));

  + if (!aHydraJobs)

  + aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks"));

  + if (!aHydraJobs)

  + throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef);

  +

  + v = *(*aHydraJobs)->value;

  +

  + } else {

  + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v);

  + }

  +

  + findJobs(state, json, autoArgs, v, "");

  + });

  + }
• replacement in src/script/hydra-eval-jobset at line 333
  [3.82]→[2.169:471](∅→∅)
  − my @cmd = (

  − "nix", "eval-hydra-jobs",

  − "--json",

  − "--gc-roots-dir", getGCRootsDir,

  − "--max-jobs", 1,

  − "--workers", $config->{evaluator_workers} // 1,

  − "--max-memory-size", $config->{evaluator_max_memory_size} // 4096,

  − "--builders", "",

  − );
  [3.82]

  [3.1424]
  + my @cmd;
• replacement in src/script/hydra-eval-jobset at line 336
  [3.125]→[2.472:586](∅→∅)
  − push @cmd,

  − ($flakeRef,

  − "--no-update-lock-file",

  − "--no-registries");
  [3.125]

  [3.281]
  + @cmd = ("hydra-eval-jobs",

  + "--flake", $flakeRef,

  + "--gc-roots-dir", getGCRootsDir,

  + "--max-jobs", 1);
• replacement in src/script/hydra-eval-jobset at line 343
  [3.434]→[2.587:721](∅→∅)
  − push @cmd,

  − ("--file", "<" . $nixExprInputName . "/" . $nixExprPath . ">",

  − inputsToArgs($inputInfo));
  [3.434]

  [3.762]
  +

  + @cmd = ("hydra-eval-jobs",

  + "<" . $nixExprInputName . "/" . $nixExprPath . ">",

  + "--gc-roots-dir", getGCRootsDir,

  + "--max-jobs", 1,

  + inputsToArgs($inputInfo));
• replacement in tests/Makefile.am at line 17
  [3.213]→[2.722:846](∅→∅)
  − PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \
  [3.213]

  [3.37753]
  + PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \