nix eval-hydra-jobs: Support job names as aggregate constituents

Fixes https://github.com/NixOS/hydra/issues/715.

flake.nix

@@ -471,21 +471,21 @@
             name = "nix-${tarball.version}";
             meta.description = "Release-critical builds";
             constituents =
-              [ tarball
+              [ "tarball"
-                build.i686-linux
+                "build.i686-linux"
-                build.x86_64-darwin
+                "build.x86_64-darwin"
-                build.x86_64-linux
+                "build.x86_64-linux"
-                build.aarch64-linux
+                "build.aarch64-linux"
-                binaryTarball.i686-linux
+                "binaryTarball.i686-linux"
-                binaryTarball.x86_64-darwin
+                "binaryTarball.x86_64-darwin"
-                binaryTarball.x86_64-linux
+                "binaryTarball.x86_64-linux"
-                binaryTarball.aarch64-linux
+                "binaryTarball.aarch64-linux"
-                tests.remoteBuilds
+                "tests.remoteBuilds"
-                tests.nix-copy-closure
+                "tests.nix-copy-closure"
-                tests.binaryTarball
+                "tests.binaryTarball"
-                #tests.evalNixpkgs
+                #"tests.evalNixpkgs"
-                #tests.evalNixOS
+                #"tests.evalNixOS"
-                installerScript
+                "installerScript"
               ];
           };
 

src/libstore/derivations.cc

@@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair)
+    const Derivation & drv, std::string_view name, RepairFlag repair)
 {
     auto references = cloneStorePathSet(drv.inputSrcs);
     for (auto & i : drv.inputDrvs)
@@ -73,8 +73,8 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    string suffix = name + drvExtension;
+    auto suffix = std::string(name) + drvExtension;
-    string contents = drv.unparse(*store, false);
+    auto contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references, repair);

src/libstore/derivations.hh

@@ -79,7 +79,7 @@ class Store;
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
+    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Store & store, const Path & drvPath);

src/nix/eval-hydra-jobs.cc

@@ -143,15 +143,23 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
                         auto a = v->attrs->get(state->symbols.create("constituents"));
                         if (!a)
                             throw EvalError("derivation must have a ‘constituents’ attribute");
+
+
                         PathSet context;
                         state->coerceToString(*a->pos, *a->value, context, true, false);
-                        PathSet drvs;
                         for (auto & i : context)
                             if (i.at(0) == '!') {
                                 size_t index = i.find("!", 1);
-                                drvs.insert(string(i, index + 1));
+                                job["constituents"].push_back(string(i, index + 1));
                             }
-                        job["constituents"] = concatStringsSep(" ", drvs);
+
+                        state->forceList(*a->value, *a->pos);
+                        for (unsigned int n = 0; n < a->value->listSize(); ++n) {
+                            auto v = a->value->listElems()[n];
+                            state->forceValue(*v);
+                            if (v->type == tString)
+                                job["namedConstituents"].push_back(state->forceStringNoCtx(*v));
+                        }
                     }
 
                     /* Register the derivation as a GC root.  !!! This
@@ -210,7 +218,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
         {
             std::set<std::string> todo{""};
             std::set<std::string> active;
-            nlohmann::json result;
+            nlohmann::json jobs;
             std::exception_ptr exc;
         };
 
@@ -295,7 +303,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
                     if (response.find("job") != response.end()) {
                         auto state(state_.lock());
                         if (json)
-                            state->result[attrPath] = response["job"];
+                            state->jobs[attrPath] = response["job"];
                         else
                             std::cout << fmt("%d: %d\n", attrPath, (std::string) response["job"]["drvPath"]);
                     }
@@ -310,7 +318,7 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
                     if (response.find("error") != response.end()) {
                         auto state(state_.lock());
                         if (json)
-                            state->result[attrPath]["error"] = response["error"];
+                            state->jobs[attrPath]["error"] = response["error"];
                         else
                             printError("error in job '%s': %s",
                                 attrPath, (std::string) response["error"]);
@@ -344,7 +352,45 @@ struct CmdEvalHydraJobs : MixJSON, MixDryRun, InstallableCommand
         if (state->exc)
             std::rethrow_exception(state->exc);
 
-        if (json) std::cout << state->result.dump(2) << "\n";
+        /* For aggregate jobs that have named consistuents
+           (i.e. constituents that are a job name rather than a
+           derivation), look up the referenced job and add it to the
+           dependencies of the aggregate derivation. */
+        for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) {
+            auto jobName = i.key();
+            auto & job = i.value();
+
+            auto named = job.find("namedConstituents");
+            if (named == job.end() || dryRun) continue;
+
+            std::string drvPath = job["drvPath"];
+            auto drv = readDerivation(*store, drvPath);
+
+            for (std::string jobName2 : *named) {
+                auto job2 = state->jobs.find(jobName2);
+                if (job2 == state->jobs.end())
+                    throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2);
+                std::string drvPath2 = (*job2)["drvPath"];
+                auto drv2 = readDerivation(*store, drvPath2);
+                job["constituents"].push_back(drvPath2);
+                drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first};
+            }
+
+            std::string drvName(store->parseStorePath(drvPath).name());
+            auto h = hashDerivationModulo(*store, drv, true);
+            auto outPath = store->makeOutputPath("out", h, drvName);
+            drv.env["out"] = store->printStorePath(outPath);
+            drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", ""));
+            auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName));
+
+            debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath);
+
+            job["drvPath"] = newDrvPath;
+            job["outputs"]["out"] = store->printStorePath(outPath);
+            job.erase("namedConstituents");
+        }
+
+        if (json) std::cout << state->jobs.dump(2) << "\n";
     }
 };