From f6890b93e868db9dfe94499550ec5e91e9a0de0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 11:24:53 +0000 Subject: [PATCH 1/3] add check-cache-status option to query wether a binary cache has the build --- src/nix-eval-jobs.cc | 45 +++++++++++++++++++++++++++++++++++++------- tests/test_eval.py | 20 ++++++++++++++++---- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 5a335a0..8544b15 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -44,6 +44,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { bool meta = false; bool showTrace = false; bool impure = false; + bool checkCacheStatus = false; size_t nrWorkers = 1; size_t maxMemorySize = 4096; @@ -93,6 +94,15 @@ struct MyArgs : MixEvalArgs, MixCommonArgs { .description = "include derivation meta field in output", .handler = {&meta, true}}); + addFlag( + {.longName = "check-cache-status", + .description = + "Check if the derivations are present locally or in " + "any configured substituters (i.e. binary cache). The " + "information " + "will be exposed in the `isCached` field of the JSON output.", + .handler = {&checkCacheStatus, true}}); + addFlag({.longName = "show-trace", .description = "print out a stack trace in case of evaluation errors", @@ -171,11 +181,26 @@ Value *topLevelValue(EvalState &state, Bindings &autoArgs) { : releaseExprTopLevelValue(state, autoArgs); } +bool queryIsCached(Store &store, std::map &outputs) { + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + + std::vector paths; + for (auto const &[key, val] : outputs) { + paths.push_back(followLinksToStorePathWithOutputs(store, val)); + } + + store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, + unknown, downloadSize, narSize); + return willBuild.empty() && unknown.empty(); +} + /* The fields of a derivation that are printed in json form */ struct Drv { std::string name; std::string system; std::string drvPath; + bool isCached; std::map outputs; std::optional meta; @@ -209,6 +234,9 @@ struct Drv { } meta = meta_; } + if (myArgs.checkCacheStatus) { + isCached = queryIsCached(*localStore, outputs); + } name = drvInfo.queryName(); system = drvInfo.querySystem(); @@ -217,15 +245,18 @@ struct Drv { }; static void to_json(nlohmann::json &json, const Drv &drv) { - json = nlohmann::json{ - {"name", drv.name}, - {"system", drv.system}, - {"drvPath", drv.drvPath}, - {"outputs", drv.outputs}, - }; + json = nlohmann::json{{"name", drv.name}, + {"system", drv.system}, + {"drvPath", drv.drvPath}, + {"outputs", drv.outputs}}; - if (drv.meta.has_value()) + if (drv.meta.has_value()) { json["meta"] = drv.meta.value(); + } + + if (myArgs.checkCacheStatus) { + json["isCached"] = drv.isCached; + } } std::string attrPathJoin(json input) { diff --git a/tests/test_eval.py b/tests/test_eval.py index 8ab26b7..5cdad3b 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -4,14 +4,14 @@ import subprocess import json from tempfile import TemporaryDirectory from pathlib import Path -from typing import List +from typing import List, Dict, Any TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") -def common_test(extra_args: List[str]) -> None: +def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: with TemporaryDirectory() as tempdir: cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args res = subprocess.run( @@ -47,14 +47,26 @@ def common_test(extra_args: List[str]) -> None: assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]["broken"] is False + return results def test_flake() -> None: - common_test(["--flake", ".#hydraJobs"]) + results = common_test(["--flake", ".#hydraJobs"]) + for result in results: + assert "isCached" not in result + + +def test_query_cache_status() -> None: + results = common_test(["--flake", ".#hydraJobs", "--check-cache-status"]) + # FIXME in the nix sandbox we cannot query binary caches, this would need some local one + for result in results: + assert "isCached" in result def test_expression() -> None: - common_test(["ci.nix"]) + results = common_test(["ci.nix"]) + for result in results: + assert "isCached" not in result with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: common_test(["-E", ci_nix.read()]) From cb8126538f1a7ee37eeae895a683d81d37305447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 11:26:15 +0000 Subject: [PATCH 2/3] fix recurseIntoAttrs handling --- src/nix-eval-jobs.cc | 85 +++++++++++++++++++++----------------------- tests/test_eval.py | 7 ++-- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/nix-eval-jobs.cc b/src/nix-eval-jobs.cc index 8544b15..68b6416 100644 --- a/src/nix-eval-jobs.cc +++ b/src/nix-eval-jobs.cc @@ -297,60 +297,55 @@ static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to, auto v = state.allocValue(); state.autoCallFunction(autoArgs, *vTmp, *v); - if (auto drvInfo = getDerivation(state, *v, false)) { + if (v->type() == nAttrs) { + if (auto drvInfo = getDerivation(state, *v, false)) { + auto drv = Drv(state, *drvInfo); + reply.update(drv); - auto drv = Drv(state, *drvInfo); - reply.update(drv); - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - if (myArgs.gcRootsDir != "") { - Path root = myArgs.gcRootsDir + "/" + - std::string(baseNameOf(drv.drvPath)); - if (!pathExists(root)) { - auto localStore = - state.store.dynamic_pointer_cast(); - auto storePath = - localStore->parseStorePath(drv.drvPath); - localStore->addPermRoot(storePath, root); + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + if (myArgs.gcRootsDir != "") { + Path root = myArgs.gcRootsDir + "/" + + std::string(baseNameOf(drv.drvPath)); + if (!pathExists(root)) { + auto localStore = + state.store + .dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drv.drvPath); + localStore->addPermRoot(storePath, root); + } } - } + } else { + auto attrs = nlohmann::json::array(); + bool recurse = + path.size() == 0; // Dont require `recurseForDerivations + // = true;` for top-level attrset - } + for (auto &i : + v->attrs->lexicographicOrder(state.symbols)) { + const std::string &name = state.symbols[i->name]; + attrs.push_back(name); - else if (v->type() == nAttrs) { - auto attrs = nlohmann::json::array(); - bool recurse = - path.size() == 0; // Dont require `recurseForDerivations = - // true;` for top-level attrset - - for (auto &i : v->attrs->lexicographicOrder(state.symbols)) { - const std::string &name = state.symbols[i->name]; - attrs.push_back(name); - - if (name == "recurseForDerivations") { - auto attrv = - v->attrs->get(state.sRecurseForDerivations); - recurse = state.forceBool(*attrv->value, attrv->pos); + if (name == "recurseForDerivations") { + auto attrv = + v->attrs->get(state.sRecurseForDerivations); + recurse = + state.forceBool(*attrv->value, attrv->pos); + } } + if (recurse) + reply["attrs"] = std::move(attrs); + else + reply["attrs"] = nlohmann::json::array(); } - if (recurse) - reply["attrs"] = std::move(attrs); - else - reply["attrs"] = nlohmann::json::array(); + } else { + // We ignore everything that cannot be build + reply["attrs"] = nlohmann::json::array(); } - - else if (v->type() == nNull) - ; - - else - throw TypeError("attribute '%s' is %s, which is not supported", - path, showType(*v)); - } catch (EvalError &e) { auto err = e.info(); - std::ostringstream oss; showErrorInfo(oss, err, loggerSettings.showTrace.get()); auto msg = oss.str(); diff --git a/tests/test_eval.py b/tests/test_eval.py index 5cdad3b..a08332e 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -23,7 +23,7 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: ) results = [json.loads(r) for r in res.stdout.split("\n") if r] - assert len(results) == 5 + assert len(results) == 4 built_job = results[0] assert built_job["attr"] == "builtJob" @@ -40,10 +40,7 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: assert recurse_drv["attr"] == "recurse.drvB" assert recurse_drv["name"] == "drvB" - recurse_recurse_bool = results[3] - assert "error" in recurse_recurse_bool - - substituted_job = results[4] + substituted_job = results[3] assert substituted_job["attr"] == "substitutedJob" assert substituted_job["name"].startswith("hello-") assert substituted_job["meta"]["broken"] is False From f8fa3ea069bf0f2547b57abf8c1aaa848def7e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 9 Sep 2022 12:32:22 +0000 Subject: [PATCH 3/3] README: update available options --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 98433c6..569b885 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,14 @@ USAGE: nix-eval-jobs [options] expr --arg Pass the value *expr* as the argument *name* to Nix functions. --argstr Pass the string *string* as the argument *name* to Nix functions. + --check-cache-status Check if the derivations are present locally or in any configured substituters (i.e. binary cache). The information will be exposed in the `isCached` field of the JSON output. --debug Set the logging verbosity level to 'debug'. --eval-store The Nix store to use for evaluations. + --expr treat the argument as a Nix expression --flake build a flake --gc-roots-dir garbage collector roots directory --help show usage information - --impure set evaluation mode + --impure allow impure expressions --include Add *path* to the list of locations used to look up `<...>` file names. --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. --max-memory-size maximum evaluation memory size @@ -61,6 +63,7 @@ USAGE: nix-eval-jobs [options] expr --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. --quiet Decrease the logging verbosity level. + --show-trace print out a stack trace in case of evaluation errors --verbose Increase the logging verbosity level. --workers number of evaluate workers ``` @@ -83,7 +86,6 @@ single large log file. In the we collect example ci configuration for various CIs. - ## Organisation of this repository On the `main` branch we target nixUnstable. When a release of nix happens, we