From 6529490cc10018d5191e50c482ac1180b96b1a3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 15:53:59 +0100 Subject: [PATCH] nix eval-hydra-jobs: Support job names as aggregate constituents Fixes https://github.com/NixOS/hydra/issues/715. --- flake.nix | 30 +++++++++---------- src/libstore/derivations.cc | 6 ++-- src/libstore/derivations.hh | 2 +- src/nix/eval-hydra-jobs.cc | 60 ++++++++++++++++++++++++++++++++----- 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/flake.nix b/flake.nix index ff9f817c0..078f51fa4 100644 --- a/flake.nix +++ b/flake.nix @@ -471,21 +471,21 @@ name = "nix-${tarball.version}"; meta.description = "Release-critical builds"; constituents = - [ tarball - build.i686-linux - build.x86_64-darwin - build.x86_64-linux - build.aarch64-linux - binaryTarball.i686-linux - binaryTarball.x86_64-darwin - binaryTarball.x86_64-linux - binaryTarball.aarch64-linux - tests.remoteBuilds - tests.nix-copy-closure - tests.binaryTarball - #tests.evalNixpkgs - #tests.evalNixOS - installerScript + [ "tarball" + "build.i686-linux" + "build.x86_64-darwin" + "build.x86_64-linux" + "build.aarch64-linux" + "binaryTarball.i686-linux" + "binaryTarball.x86_64-darwin" + "binaryTarball.x86_64-linux" + "binaryTarball.aarch64-linux" + "tests.remoteBuilds" + "tests.nix-copy-closure" + "tests.binaryTarball" + #"tests.evalNixpkgs" + #"tests.evalNixOS" + "installerScript" ]; }; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a554cb66d..5dba84aaf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair) + const Derivation & drv, std::string_view name, RepairFlag repair) { auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) @@ -73,8 +73,8 @@ StorePath writeDerivation(ref store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(*store, false); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..7222d25e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -79,7 +79,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath); diff --git a/src/nix/eval-hydra-jobs.cc b/src/nix/eval-hydra-jobs.cc index 180c401b2..b04e6df7b 100644 --- a/src/nix/eval-hydra-jobs.cc +++ b/src/nix/eval-hydra-jobs.cc @@ -143,15 +143,23 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand auto a = v->attrs->get(state->symbols.create("constituents")); if (!a) throw EvalError("derivation must have a ‘constituents’ attribute"); + + PathSet context; state->coerceToString(*a->pos, *a->value, context, true, false); - PathSet drvs; for (auto & i : context) if (i.at(0) == '!') { size_t index = i.find("!", 1); - drvs.insert(string(i, index + 1)); + job["constituents"].push_back(string(i, index + 1)); } - job["constituents"] = concatStringsSep(" ", drvs); + + state->forceList(*a->value, *a->pos); + for (unsigned int n = 0; n < a->value->listSize(); ++n) { + auto v = a->value->listElems()[n]; + state->forceValue(*v); + if (v->type == tString) + job["namedConstituents"].push_back(state->forceStringNoCtx(*v)); + } } /* Register the derivation as a GC root. !!! This @@ -210,7 +218,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand { std::set todo{""}; std::set active; - nlohmann::json result; + nlohmann::json jobs; std::exception_ptr exc; }; @@ -295,7 +303,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand if (response.find("job") != response.end()) { auto state(state_.lock()); if (json) - state->result[attrPath] = response["job"]; + state->jobs[attrPath] = response["job"]; else std::cout << fmt("%d: %d\n", attrPath, (std::string) response["job"]["drvPath"]); } @@ -310,7 +318,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand if (response.find("error") != response.end()) { auto state(state_.lock()); if (json) - state->result[attrPath]["error"] = response["error"]; + state->jobs[attrPath]["error"] = response["error"]; else printError("error in job '%s': %s", attrPath, (std::string) response["error"]); @@ -344,7 +352,45 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand if (state->exc) std::rethrow_exception(state->exc); - if (json) std::cout << state->result.dump(2) << "\n"; + /* For aggregate jobs that have named consistuents + (i.e. constituents that are a job name rather than a + derivation), look up the referenced job and add it to the + dependencies of the aggregate derivation. */ + for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { + auto jobName = i.key(); + auto & job = i.value(); + + auto named = job.find("namedConstituents"); + if (named == job.end() || dryRun) continue; + + std::string drvPath = job["drvPath"]; + auto drv = readDerivation(*store, drvPath); + + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + auto drv2 = readDerivation(*store, drvPath2); + job["constituents"].push_back(drvPath2); + drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first}; + } + + std::string drvName(store->parseStorePath(drvPath).name()); + auto h = hashDerivationModulo(*store, drv, true); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", "")); + auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); + + debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); + job.erase("namedConstituents"); + } + + if (json) std::cout << state->jobs.dump(2) << "\n"; } };