grahamc/hydra2 - Change 5MP35ORVVILU3GXYITYXD754QFZCOZ3X45QO54K2GW5UKNHX66SAC

hydra-eval-jobs: Ugly hackery to reduce memory usage

You can now set 'evaluator_max_heap_size' to make hydra-eval-jobs restart itself if the Boehm heap exceeds the specified size.

For example, with 'evaluator_max_heap_size = 256000000',

$ hydra-eval-jobs '<nixpkgs/pkgs/top-level/release.nix>' -I nixpkgs=channel:nixos-17.09

has a max RSS of .56 GiB rather than 4.7 GiB.

Unfortunately it doesn't help much for the NixOS jobsets because of the "tested" job which requires a huge amount of memory all by itself.

Created by Eelco Dolstra on June 5, 2018

5MP35ORVVILU3GXYITYXD754QFZCOZ3X45QO54K2GW5UKNHX66SAC

Dependencies

In channels

main

Change contents

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 18 [6.303]
[6.452]
[6.452]
```
#include <sys/types.h>
#include <sys/wait.h>
```

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 55 [6.303]

[7.1407]


static std::string lastAttrPath;
static bool comma = false;
static size_t maxHeapSize;
struct BailOut { };

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 65 [6.303]

[8.1574]

[9.369]

bool lte(const std::string & s1, const std::string & s2)
{
    size_t p1 = 0, p2 = 0;
    while (true) {
        if (p1 == s1.size()) return p2 == s2.size();
        if (p2 == s2.size()) return true;
        auto d1 = s1.find('.', p1);
        auto d2 = s2.find('.', p2);
        auto c = s1.compare(p1, d1 - p1, s2, p2, d2 - p2);
        if (c < 0) return true;
        if (c > 0) return false;
        p1 = d1 == std::string::npos ? s1.size() : d1 + 1;
        p2 = d2 == std::string::npos ? s2.size() : d2 + 1;
    }
}

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 90 [6.303]
[10.257]
[7.1529]
```
    if (lastAttrPath != "" && lte(attrPath, lastAttrPath)) return;
```
Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 110 [6.303]
[11.123]
[11.123]
```
            if (comma) { std::cout << ","; comma = false; }
```

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 157 [6.303]

[12.16]

            }
            GC_prof_stats_s gc;
            GC_get_prof_stats(&gc, sizeof(gc));
            if (gc.heapsize_full > maxHeapSize) {
                printInfo("restarting hydra-eval-jobs after job '%s' because heap size is at %d bytes", attrPath, gc.heapsize_full);
                lastAttrPath = attrPath;
                throw BailOut();

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 171 [6.303]

B:BD[13.120] → [14.502:544]

B:BD[14.544] → [15.219:280]

∅:D[15.280] → [14.605:690]

B:BD[14.605] → [14.605:690]

                for (auto & i : *v.attrs)
                    findJobs(state, top, autoArgs, *i.value,
                        (attrPath.empty() ? "" : attrPath + ".") + (string) i.name);

[13.120]

[13.336]

                for (auto & i : v.attrs->lexicographicOrder()) {
                    std::string name(i->name);
                    /* Skip jobs with dots in the name. */
                    if (name.find('.') != std::string::npos) {
                        printError("skipping job with illegal name '%s'", name);
                        continue;
                    }
                    findJobs(state, top, autoArgs, *i->value,
                        (attrPath.empty() ? "" : attrPath + ".") + name);
                }

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 202 [6.303]
B:BD[16.30] → [17.0:10]
```
        {
```
[16.30]
[2.411]
```
        if (comma) { std::cout << ","; comma = false; }
```
Deletion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 205 [6.303]
B:BD[9.1824] → [17.11:21]
```
        }
```

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 211 [6.303]

[6.570]

[18.0]

    assert(lte("abc", "def"));
    assert(lte("abc", "def.foo"));
    assert(!lte("def", "abc"));
    assert(lte("nixpkgs.hello", "nixpkgs"));
    assert(lte("nixpkgs.hello", "nixpkgs.hellooo"));
    assert(lte("gitAndTools.git-annex.x86_64-darwin", "gitAndTools.git-annex.x86_64-linux"));
    assert(lte("gitAndTools.git-annex.x86_64-linux", "gitAndTools.git-annex-remote-b2.aarch64-linux"));

Insertion in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 231 [6.303]

[4.336]

[19.80]

        maxHeapSize = config->getIntOption("evaluator_max_heap_size", 1UL << 30);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 257 [6.303]

B:BD[20.1] → [21.0:119]

B:BD[21.119] → [15.722:754]

        /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
        settings.builders = "";

[20.1]

[21.158]

        JSONObject json(std::cout, true);
        std::cout.flush();

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 260 [6.303]

∅:D[21.159] → [20.1:105]

B:BD[20.1] → [20.1:105]

B:BD[20.105] → [5.0:42]

        /* Prevent access to paths outside of the Nix search path and
           to the environment. */
        evalSettings.restrictEval = true;

[21.159]

[6.854]

        do {
            Pipe pipe;
            pipe.create();

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 265 [6.303]

B:BD[6.855] → [19.1196:1272]

        if (releaseExpr == "") throw UsageError("no expression specified");

[6.855]

[22.808]

            ProcessOptions options;
            options.allowVfork = false;

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 268 [6.303]

B:BD[22.809] → [19.1273:1366]

        if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");

[22.809]

[22.810]

            auto pid = startProcess([&]() {
                pipe.readSide = -1;

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 271 [6.303]

B:BD[22.811] → [15.794:851]

        EvalState state(myArgs.searchPath, openStore());

[22.811]

[6.880]

                if (lastAttrPath != "") debug("resuming from '%s'", lastAttrPath);

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 273 [6.303]

B:BD[6.881] → [15.852:910]

        Bindings & autoArgs = *myArgs.getAutoArgs(state);

[6.881]

[10.1624]

                /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */
                settings.builders = "";

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 276 [6.303]

B:BD[23.1] → [19.1927:1944]

B:BD[19.1944] → [3.37:99]

        Value v;
        state.evalFile(lookupFileArg(state, releaseExpr), v);

[10.1625]

[6.1021]

                /* Prevent access to paths outside of the Nix search path and
                   to the environment. */
                evalSettings.restrictEval = true;

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 280 [6.303]

B:BD[6.1022] → [2.453:495]

∅:D[2.495] → [9.1886:1934]

B:BD[9.1886] → [9.1886:1934]

        JSONObject json(std::cout, true);
        findJobs(state, json, autoArgs, v, "");

[6.1022]

                if (releaseExpr == "") throw UsageError("no expression specified");

Replacement in src/hydra-eval-jobs/hydra-eval-jobs.cc at line 282 [6.303]

B:BD[6.1023] → [19.2127:2155]

        state.printStats();

[6.1023]

[19.2155]

                if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified");
                EvalState state(myArgs.searchPath, openStore());
                Bindings & autoArgs = *myArgs.getAutoArgs(state);
                Value v;
                state.evalFile(lookupFileArg(state, releaseExpr), v);
                comma = lastAttrPath != "";
                try {
                    findJobs(state, json, autoArgs, v, "");
                    lastAttrPath = "";
                } catch (BailOut &) { }
                writeFull(pipe.writeSide.get(), lastAttrPath);
                exit(0);
            }, options);
            pipe.writeSide = -1;
            int status;
            while (true) {
                checkInterrupt();
                if (waitpid(pid, &status, 0) == pid) break;
                if (errno != EINTR) continue;
            }
            if (status != 0)
                throw Exit(WIFEXITED(status) ? WEXITSTATUS(status) : 99);
            lastAttrPath = drainFD(pipe.readSide.get());
        } while (lastAttrPath != "");