Add hydra.conf option "nar_buffer_size" to configure memoryTokens limit

[?]

Mar 3, 2017, 11:37 AM

NSBNNM77LKRFLORJOW4NB2WXDWHC3MKRNLGRCLUKKZZXWZK7TJFAC

Dependencies

[2] 74WP3O3F max-output-size -> max_output_size
[3] A4A5KO5Q Add error/warnings for deprecated store specification
[4] 5CNQUQ77 Bump Nix
[5] EPWEMRI2 Allow determinism checking for entire jobsets
[6] UUN5WH4D Decrease memoryTokens
[7] SL3WSRAC hydra-queue-runner: Limit memory usage
[8] 7VQ4ALFY Update "make check" for the new queue runner
[9] SJQC2I3N Bump memory limit a bit
[10] 24BMQDZA Start of single-process hydra-queue-runner
[11] NJJ7H64S Very basic multi-threaded queue runner
[12] RQUAATWB Add status dump facility
[13] GJV2J5HX Pool local store connections
[14] RGCPOSHY hydra-queue-runner: Ensure regular status dumps
[15] XCHAKH4D hydra-queue-runner: Bump memory limit to reflect more accurate accounting
[16] PQFOMNTL hydra-queue-runner: More stats
[17] OG3Z3QGC Namespace cleanup
[18] SOB276BA Keep some statistics for the binary cache stores
[19] BRAESISH Warn if PostgreSQL appears stalled
[20] ENXUSMSV Make concurrency more robust
[21] CNLNT3T4 Allow only 1 thread to send a closure to a given machine at the same time
[22] V2UCCYN3 hydra-queue-runner: Get store mode configuration from hydra.conf
[23] IKJBYIGY Update to reflect BinaryCacheStore changes
[24] OCZ4LSGG Automatically retry aborted builds
[25] PDI5YPBY Use store-api for binary cache instantiation
[26] 5AIYUMTB Basic remote building
[27] BG6PEOB2 Make the output size limit configurable
[28] HJOEIMLR Refactor
[29] B2L4T3X6 Sync with Nix
[30] EOO4EFWD Use a single BinaryCacheStore for all threads
[31] PLOZBRTR Add command ‘hydra-queue-runner --status’ to show current status
[*] MHVIT4JY Split hydra-queue-runner.cc more
[*] MB3TISH2 Rate-limit the number of threads copying closures at the same time

Change contents

replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 20

[7.7486]→[7.7486:7501](∅→∅),[7.7501]→[6.0:65](∅→∅)

State::State()
: memoryTokens(12ULL << 30) // FIXME: make this configurable

[7.22]

[7.7501]

struct Config
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 22

[7.115]→[7.7737:7840](∅→∅),[7.7503]→[7.7737:7840](∅→∅)

hydraData = getEnv("HYDRA_DATA");
if (hydraData == "") throw Error("$HYDRA_DATA must be set");

[7.7503]

[7.0]

std::map<std::string, std::string> options;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 24

[7.1]→[7.1:118](∅→∅)

/* Read hydra.conf. */
auto hydraConfigFile = getEnv("HYDRA_CONFIG");
if (pathExists(hydraConfigFile)) {

[7.1]

[7.118]

Config()
{
/* Read hydra.conf. */
auto hydraConfigFile = getEnv("HYDRA_CONFIG");
if (pathExists(hydraConfigFile)) {
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 30

[7.119]→[7.119:262](∅→∅)

for (auto line : tokenizeString<Strings>(readFile(hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#')));

[7.119]

[7.262]

for (auto line : tokenizeString<Strings>(readFile(hydraConfigFile), "\n")) {
line = trim(string(line, 0, line.find('#')));
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 33

[7.263]→[7.263:352](∅→∅)

auto eq = line.find('=');
if (eq == std::string::npos) continue;

[7.263]

[7.352]

auto eq = line.find('=');
if (eq == std::string::npos) continue;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 36

[7.353]→[7.353:466](∅→∅)

auto key = trim(std::string(line, 0, eq));
auto value = trim(std::string(line, eq + 1));

[7.353]

[7.466]

auto key = trim(std::string(line, 0, eq));
auto value = trim(std::string(line, eq + 1));
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 39

[7.467]→[7.467:504](∅→∅)

if (key == "") continue;

[7.467]

[7.7840]

if (key == "") continue;
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 41

[7.7841]→[7.505:543](∅→∅)

hydraConfig[key] = value;

[7.7841]

[7.543]

options[key] = value;
}
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 46

[7.560]

[7.3819]

std::string getStrOption(const std::string & key, const std::string & def = "")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 48

[7.3825]→[2.0:56](∅→∅),[2.56]→[7.3881:3932](∅→∅),[7.3881]→[7.3881:3932](∅→∅)

std::string s = hydraConfig["max_output_size"];
if (s != "") string2Int(s, maxOutputSize);

[7.3825]

[7.3932]

auto i = options.find(key);
return i == options.end() ? def : i->second;
}

uint64_t getIntOption(const std::string & key, uint64_t def = 0)
{
auto i = options.find(key);
return i == options.end() ? def : std::stoi(i->second);
}

bool getBoolOption(const std::string & key, bool def = false)
{
auto i = options.find(key);
return i == options.end() ? def : i->second == "true";
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 63

[7.3938]

[7.3938]

};

static uint64_t getMemSize()
{
auto pages = sysconf(_SC_PHYS_PAGES);
return pages >= 0 ? pages * sysconf(_SC_PAGESIZE) : 4ULL << 30;
}

State::State()
: config(std::make_unique<Config>())
, memoryTokens(config->getIntOption("nar_buffer_size", getMemSize() / 2))
, maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30))
{
printInfo("using %d bytes for the NAR buffer", memoryTokens.capacity());

hydraData = getEnv("HYDRA_DATA");
if (hydraData == "") throw Error("$HYDRA_DATA must be set");
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 86

[3.49]→[3.49:90](∅→∅)

if (hydraConfig["store_mode"] != "")

[3.49]

[3.90]

if (config->getStrOption("store_mode") != "")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 88

[3.175]→[3.175:222](∅→∅)

if (hydraConfig["binary_cache_dir"] != "")

[3.175]

[3.222]

if (config->getStrOption("binary_cache_dir") != "")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 90

[3.339]→[3.339:392](∅→∅)

if (hydraConfig["binary_cache_s3_bucket"] != "")

[3.339]

[3.392]

if (config->getStrOption("binary_cache_s3_bucket") != "")
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 92

[3.513]→[3.513:572](∅→∅)

if (hydraConfig["binary_cache_secret_key_file"] != "")

[3.513]

[3.572]

if (config->getStrOption("binary_cache_secret_key_file") != "")
edit in src/hydra-queue-runner/hydra-queue-runner.cc at line 835

[7.264]→[7.4619:4620](∅→∅),[7.609]→[7.4619:4620](∅→∅),[7.859]→[7.4619:4620](∅→∅),[7.4619]→[7.4619:4620](∅→∅),[7.4620]→[4.293:409](∅→∅)

_destStore = hydraConfig["store_uri"] == ""
? localStore
: openStore(hydraConfig["store_uri"]);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 836

[7.300]→[7.300:393](∅→∅)

auto isTrue = [](const std::string & s) {
return s == "1" || s == "true";
};

[7.300]

[7.393]

auto storeUri = config->getStrOption("store_uri");
_destStore = storeUri == "" ? localStore : openStore(storeUri);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 839

[7.394]→[7.394:455](∅→∅)

useSubstitutes = isTrue(hydraConfig["use-substitutes"]);

[7.394]

[7.1249]

useSubstitutes = config->getBoolOption("use-substitutes", false);
replacement in src/hydra-queue-runner/hydra-queue-runner.cc at line 842

[5.2091]→[5.2091:2173](∅→∅)

for (auto & s : tokenizeString<Strings>(hydraConfig["xxx-jobset-repeats"])) {

[5.2091]

[5.2173]

for (auto & s : tokenizeString<Strings>(config->getStrOption("xxx-jobset-repeats"))) {
edit in src/hydra-queue-runner/state.hh at line 251

[7.4331]

[7.4331]
edit in src/hydra-queue-runner/state.hh at line 253

[7.4332]

[7.4332]

class Config;
edit in src/hydra-queue-runner/state.hh at line 255

[7.4333]

[7.4333]
edit in src/hydra-queue-runner/state.hh at line 260

[33.33065]

[33.33065]

std::unique_ptr<Config> config;
edit in src/hydra-queue-runner/state.hh at line 270

[7.1818]→[7.1818:1870](∅→∅),[7.1870]→[7.1915:1916](∅→∅)

std::map<std::string, std::string> hydraConfig;
replacement in src/hydra-queue-runner/state.hh at line 413

[7.1503]→[7.4313:4352](∅→∅)

size_t maxOutputSize = 2ULL << 30;

[7.1503]

[7.4352]

size_t maxOutputSize;
edit in src/hydra-queue-runner/token-server.hh at line 102

[34.3312]

[34.3312]

size_t capacity()
{
return maxTokens;
}