nix eval-hydra-jobs: Support job names as aggregate constituents

Fixes https://github.com/NixOS/hydra/issues/715.
This commit is contained in:
Eelco Dolstra 2020-02-17 15:53:59 +01:00
parent 8a78bcf6a2
commit 6529490cc1
4 changed files with 72 additions and 26 deletions

View file

@ -471,21 +471,21 @@
name = "nix-${tarball.version}"; name = "nix-${tarball.version}";
meta.description = "Release-critical builds"; meta.description = "Release-critical builds";
constituents = constituents =
[ tarball [ "tarball"
build.i686-linux "build.i686-linux"
build.x86_64-darwin "build.x86_64-darwin"
build.x86_64-linux "build.x86_64-linux"
build.aarch64-linux "build.aarch64-linux"
binaryTarball.i686-linux "binaryTarball.i686-linux"
binaryTarball.x86_64-darwin "binaryTarball.x86_64-darwin"
binaryTarball.x86_64-linux "binaryTarball.x86_64-linux"
binaryTarball.aarch64-linux "binaryTarball.aarch64-linux"
tests.remoteBuilds "tests.remoteBuilds"
tests.nix-copy-closure "tests.nix-copy-closure"
tests.binaryTarball "tests.binaryTarball"
#tests.evalNixpkgs #"tests.evalNixpkgs"
#tests.evalNixOS #"tests.evalNixOS"
installerScript "installerScript"
]; ];
}; };

View file

@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const
StorePath writeDerivation(ref<Store> store, StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair) const Derivation & drv, std::string_view name, RepairFlag repair)
{ {
auto references = cloneStorePathSet(drv.inputSrcs); auto references = cloneStorePathSet(drv.inputSrcs);
for (auto & i : drv.inputDrvs) for (auto & i : drv.inputDrvs)
@ -73,8 +73,8 @@ StorePath writeDerivation(ref<Store> store,
/* Note that the outputs of a derivation are *not* references /* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be (that can be missing (of course) and should not necessarily be
held during a garbage collection). */ held during a garbage collection). */
string suffix = name + drvExtension; auto suffix = std::string(name) + drvExtension;
string contents = drv.unparse(*store, false); auto contents = drv.unparse(*store, false);
return settings.readOnlyMode return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references) ? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair); : store->addTextToStore(suffix, contents, references, repair);

View file

@ -79,7 +79,7 @@ class Store;
/* Write a derivation to the Nix store, and return its path. */ /* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store, StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair); const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */ /* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath); Derivation readDerivation(const Store & store, const Path & drvPath);

View file

@ -143,15 +143,23 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
auto a = v->attrs->get(state->symbols.create("constituents")); auto a = v->attrs->get(state->symbols.create("constituents"));
if (!a) if (!a)
throw EvalError("derivation must have a constituents attribute"); throw EvalError("derivation must have a constituents attribute");
PathSet context; PathSet context;
state->coerceToString(*a->pos, *a->value, context, true, false); state->coerceToString(*a->pos, *a->value, context, true, false);
PathSet drvs;
for (auto & i : context) for (auto & i : context)
if (i.at(0) == '!') { if (i.at(0) == '!') {
size_t index = i.find("!", 1); size_t index = i.find("!", 1);
drvs.insert(string(i, index + 1)); job["constituents"].push_back(string(i, index + 1));
}
state->forceList(*a->value, *a->pos);
for (unsigned int n = 0; n < a->value->listSize(); ++n) {
auto v = a->value->listElems()[n];
state->forceValue(*v);
if (v->type == tString)
job["namedConstituents"].push_back(state->forceStringNoCtx(*v));
} }
job["constituents"] = concatStringsSep(" ", drvs);
} }
/* Register the derivation as a GC root. !!! This /* Register the derivation as a GC root. !!! This
@ -210,7 +218,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
{ {
std::set<std::string> todo{""}; std::set<std::string> todo{""};
std::set<std::string> active; std::set<std::string> active;
nlohmann::json result; nlohmann::json jobs;
std::exception_ptr exc; std::exception_ptr exc;
}; };
@ -295,7 +303,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
if (response.find("job") != response.end()) { if (response.find("job") != response.end()) {
auto state(state_.lock()); auto state(state_.lock());
if (json) if (json)
state->result[attrPath] = response["job"]; state->jobs[attrPath] = response["job"];
else else
std::cout << fmt("%d: %d\n", attrPath, (std::string) response["job"]["drvPath"]); std::cout << fmt("%d: %d\n", attrPath, (std::string) response["job"]["drvPath"]);
} }
@ -310,7 +318,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
if (response.find("error") != response.end()) { if (response.find("error") != response.end()) {
auto state(state_.lock()); auto state(state_.lock());
if (json) if (json)
state->result[attrPath]["error"] = response["error"]; state->jobs[attrPath]["error"] = response["error"];
else else
printError("error in job '%s': %s", printError("error in job '%s': %s",
attrPath, (std::string) response["error"]); attrPath, (std::string) response["error"]);
@ -344,7 +352,45 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
if (state->exc) if (state->exc)
std::rethrow_exception(state->exc); std::rethrow_exception(state->exc);
if (json) std::cout << state->result.dump(2) << "\n"; /* For aggregate jobs that have named consistuents
(i.e. constituents that are a job name rather than a
derivation), look up the referenced job and add it to the
dependencies of the aggregate derivation. */
for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) {
auto jobName = i.key();
auto & job = i.value();
auto named = job.find("namedConstituents");
if (named == job.end() || dryRun) continue;
std::string drvPath = job["drvPath"];
auto drv = readDerivation(*store, drvPath);
for (std::string jobName2 : *named) {
auto job2 = state->jobs.find(jobName2);
if (job2 == state->jobs.end())
throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
std::string drvPath2 = (*job2)["drvPath"];
auto drv2 = readDerivation(*store, drvPath2);
job["constituents"].push_back(drvPath2);
drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first};
}
std::string drvName(store->parseStorePath(drvPath).name());
auto h = hashDerivationModulo(*store, drv, true);
auto outPath = store->makeOutputPath("out", h, drvName);
drv.env["out"] = store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", ""));
auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath);
job["drvPath"] = newDrvPath;
job["outputs"]["out"] = store->printStorePath(outPath);
job.erase("namedConstituents");
}
if (json) std::cout << state->jobs.dump(2) << "\n";
} }
}; };