From 9ba4417940ffdd0fadea43f68c61ef948a4b8d39 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 16:05:50 -0500
Subject: [PATCH 01/10] Prepare for CA derivation support with lower impact
 changes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is just C++ changes without any Perl / Frontend / SQL Schema
changes.

The idea is that it should be possible to redeploy Hydra with these
chnages with (a) no schema migration and also (b) no regressions. We
should be able to much more safely deploy these to a staging server and
then production `hydra.nixos.org`.

Extracted from #875

Co-Authored-By: Théophane Hufschmitt <theophane.hufschmitt@tweag.io>
Co-Authored-By: Alexander Sosedkin <monk@unboiled.info>
Co-Authored-By: Andrea Ciceri <andrea.ciceri@autistici.org>
Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs>
Co-Authored-By: Sandro Jäckel <sandro.jaeckel@gmail.com>
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc       |  16 ++-
 src/hydra-queue-runner/build-remote.cc       | 105 +++++++++++---
 src/hydra-queue-runner/build-result.cc       |  19 ++-
 src/hydra-queue-runner/builder.cc            |   8 +-
 src/hydra-queue-runner/hydra-build-result.hh |   4 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |  27 +++-
 src/hydra-queue-runner/queue-monitor.cc      | 137 ++++++++++---------
 src/hydra-queue-runner/state.hh              |   4 +-
 8 files changed, 213 insertions(+), 107 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index 2fe2c80f..d007189d 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -178,7 +178,11 @@ static void worker(
 
             if (auto drv = getDerivation(state, *v, false)) {
 
-                DrvInfo::Outputs outputs = drv->queryOutputs();
+                // CA derivations do not have static output paths, so we
+                // have to defensively not query output paths in case we
+                // encounter one.
+                DrvInfo::Outputs outputs = drv->queryOutputs(
+                    !experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
 
                 if (drv->querySystem() == "unknown")
                     throw EvalError("derivation must have a 'system' attribute");
@@ -239,12 +243,12 @@ static void worker(
                 }
 
                 nlohmann::json out;
-                for (auto & j : outputs)
-                    // FIXME: handle CA/impure builds.
-                    if (j.second)
-                        out[j.first] = state.store->printStorePath(*j.second);
+                for (auto & [outputName, optOutputPath] : outputs)
+                    if (optOutputPath)
+                        out[outputName] = state.store->printStorePath(*optOutputPath);
+                    else
+                        out[outputName] = "";
                 job["outputs"] = std::move(out);
-
                 reply["job"] = std::move(job);
             }
 
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 73f46a53..26f6d63f 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -182,6 +182,41 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
     return sorted;
 }
 
+/**
+ * Replace the input derivations by their output paths to send a minimal closure
+ * to the builder.
+ *
+ * If we can afford it, resolve it, so that the newly generated derivation still
+ * has some sensible output paths.
+ */
+BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
+{
+    BasicDerivation ret;
+    auto outputHashes = staticOutputHashes(store, drv);
+    if (!drv.type().hasKnownOutputPaths()) {
+        auto maybeBasicDrv = drv.tryResolve(store);
+        if (!maybeBasicDrv)
+            throw Error(
+                "the derivation '%s' can’t be resolved. It’s probably "
+                "missing some outputs",
+                store.printStorePath(drvPath));
+        ret = *maybeBasicDrv;
+    } else {
+        // If the derivation is a real `InputAddressed` derivation, we must
+        // resolve it manually to keep the original output paths
+        ret = BasicDerivation(drv);
+        for (auto & [drvPath, node] : drv.inputDrvs.map) {
+            auto drv2 = store.readDerivation(drvPath);
+            auto drv2Outputs = drv2.outputsAndOptPaths(store);
+            for (auto & name : node.value) {
+                auto inputPath = drv2Outputs.at(name);
+                ret.inputSrcs.insert(*inputPath.second);
+            }
+        }
+    }
+    return ret;
+}
+
 static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
     std::string base(drvPath.to_string());
@@ -228,17 +263,7 @@ static BasicDerivation sendInputs(
     counter & nrStepsCopyingTo
 )
 {
-    BasicDerivation basicDrv(*step.drv);
-
-    for (const auto & [drvPath, node] : step.drv->inputDrvs.map) {
-        auto drv2 = localStore.readDerivation(drvPath);
-        for (auto & name : node.value) {
-            if (auto i = get(drv2.outputs, name)) {
-                auto outPath = i->path(localStore, drv2.name, name);
-                basicDrv.inputSrcs.insert(*outPath);
-            }
-        }
-    }
+    BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath);
 
     /* Ensure that the inputs exist in the destination store. This is
        a no-op for regular stores, but for the binary cache store,
@@ -323,8 +348,33 @@ static BuildResult performBuild(
             result.stopTime = stop;
         }
     }
+
+    // Get the newly built outputs, either from the remote if it
+    // supports it, or by introspecting the derivation if the remote is
+    // too old.
     if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 6) {
-        ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
+        auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(localStore, conn);
+        for (auto && [output, realisation] : builtOutputs)
+            result.builtOutputs.insert_or_assign(
+                std::move(output.outputName),
+                std::move(realisation));
+    } else {
+        // If the remote is too old to handle CA derivations, we can’t get this
+        // far anyways
+        assert(drv.type().hasKnownOutputPaths());
+        DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore);
+        auto outputHashes = staticOutputHashes(localStore, drv);
+        for (auto & [outputName, output] : drvOutputs) {
+            auto outputPath = output.second;
+            // We’ve just asserted that the output paths of the derivation
+            // were known
+            assert(outputPath);
+            auto outputHash = outputHashes.at(outputName);
+            auto drvOutput = DrvOutput { outputHash, outputName };
+            result.builtOutputs.insert_or_assign(
+                std::move(outputName),
+                Realisation { drvOutput, *outputPath });
+        }
     }
 
     return result;
@@ -376,6 +426,7 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
+    for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the
           NAR that getBuildOutput() needs. */
@@ -395,7 +446,8 @@ static void copyPathFromRemote(
           extractNarData(tee, localStore.printStorePath(info.path), narMembers);
       });
 
-      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
+      store->addToStore(info, *source2, NoRepair, NoCheckSigs);
+    }
 }
 
 static void copyPathsFromRemote(
@@ -592,6 +644,10 @@ void State::buildRemote(ref<Store> destStore,
             result.logFile = "";
         }
 
+        StorePathSet outputs;
+        for (auto & [_, realisation] : buildResult.builtOutputs)
+            outputs.insert(realisation.outPath);
+
         /* Copy the output paths. */
         if (!machine->isLocalhost() || localStore != std::shared_ptr<Store>(destStore)) {
             updateStep(ssReceivingOutputs);
@@ -600,12 +656,6 @@ void State::buildRemote(ref<Store> destStore,
 
             auto now1 = std::chrono::steady_clock::now();
 
-            StorePathSet outputs;
-            for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
-                if (i.second.second)
-                   outputs.insert(*i.second.second);
-            }
-
             size_t totalNarSize = 0;
             auto infos = build_remote::queryPathInfos(conn, *localStore, outputs, totalNarSize);
 
@@ -624,6 +674,23 @@ void State::buildRemote(ref<Store> destStore,
             result.overhead += std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
         }
 
+        /* Register the outputs of the newly built drv */
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            auto outputHashes = staticOutputHashes(*localStore, *step->drv);
+            for (auto & [outputName, realisation] : buildResult.builtOutputs) {
+                // Register the resolved drv output
+                localStore->registerDrvOutput(realisation);
+                destStore->registerDrvOutput(realisation);
+
+                // Also register the unresolved one
+                auto unresolvedRealisation = realisation;
+                unresolvedRealisation.signatures.clear();
+                unresolvedRealisation.id.drvHash = outputHashes.at(outputName);
+                localStore->registerDrvOutput(unresolvedRealisation);
+                destStore->registerDrvOutput(unresolvedRealisation);
+            }
+        }
+
         /* Shut down the connection. */
         child.to = -1;
         child.pid.wait();
diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index 691c1f19..ffdc37b7 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -11,18 +11,18 @@ using namespace nix;
 BuildOutput getBuildOutput(
     nix::ref<Store> store,
     NarMemberDatas & narMembers,
-    const Derivation & drv)
+    const OutputPathMap derivationOutputs)
 {
     BuildOutput res;
 
     /* Compute the closure size. */
     StorePathSet outputs;
     StorePathSet closure;
-    for (auto & i : drv.outputsAndOptPaths(*store))
-        if (i.second.second) {
-            store->computeFSClosure(*i.second.second, closure);
-            outputs.insert(*i.second.second);
-        }
+    for (auto& [outputName, outputPath] : derivationOutputs) {
+        store->computeFSClosure(outputPath, closure);
+        outputs.insert(outputPath);
+        res.outputs.insert({outputName, outputPath});
+    }
     for (auto & path : closure) {
         auto info = store->queryPathInfo(path);
         res.closureSize += info->narSize;
@@ -107,13 +107,12 @@ BuildOutput getBuildOutput(
     /* If no build products were explicitly declared, then add all
        outputs as a product of type "nix-build". */
     if (!explicitProducts) {
-        for (auto & [name, output] : drv.outputs) {
+        for (auto & [name, output] : derivationOutputs) {
             BuildProduct product;
-            auto outPath = output.path(*store, drv.name, name);
-            product.path = store->printStorePath(*outPath);
+            product.path = store->printStorePath(output);
             product.type = "nix-build";
             product.subtype = name == "out" ? "" : name;
-            product.name = outPath->name();
+            product.name = output.name();
 
             auto file = narMembers.find(product.path);
             assert(file != narMembers.end());
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 307eee8e..42983941 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         if (result.stepStatus == bsSuccess) {
             updateStep(ssPostProcessing);
-            res = getBuildOutput(destStore, narMembers, *step->drv);
+            res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath));
         }
     }
 
@@ -277,9 +277,9 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         assert(stepNr);
 
-        for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
-            if (i.second.second)
-               addRoot(*i.second.second);
+        for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) {
+            if (i.second)
+               addRoot(*i.second);
         }
 
         /* Register success in the database for all Build objects that
diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh
index a3f71ae9..7d47f67c 100644
--- a/src/hydra-queue-runner/hydra-build-result.hh
+++ b/src/hydra-queue-runner/hydra-build-result.hh
@@ -36,10 +36,12 @@ struct BuildOutput
 
     std::list<BuildProduct> products;
 
+    std::map<std::string, nix::StorePath> outputs;
+
     std::map<std::string, BuildMetric> metrics;
 };
 
 BuildOutput getBuildOutput(
     nix::ref<nix::Store> store,
     NarMemberDatas & narMembers,
-    const nix::Derivation & drv);
+    const nix::OutputPathMap derivationOutputs);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1d54bb93..2f2c6091 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -312,10 +312,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 
     if (r.affected_rows() == 0) goto restart;
 
-    for (auto & [name, output] : step->drv->outputs)
+    for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
-            buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
+            buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
 
     if (status == bsBusy)
         txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@@ -352,11 +352,23 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
     assert(result.logFile.find('\t') == std::string::npos);
     txn.exec(fmt("notify step_finished, '%d\t%d\t%s'",
             buildId, stepNr, result.logFile));
+
+    if (result.stepStatus == bsSuccess) {
+        // Update the corresponding `BuildStepOutputs` row to add the output path
+        auto res = txn.exec_params1("select drvPath from BuildSteps where build = $1 and stepnr = $2", buildId, stepNr);
+        assert(res.size());
+        StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
+        // If we've finished building, all the paths should be known
+        for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath))
+            txn.exec_params0
+                ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
+                  buildId, stepNr, name, localStore->printStorePath(output));
+    }
 }
 
 
 int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-    Build::ptr build, const StorePath & drvPath, const std::string & outputName, const StorePath & storePath)
+    Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath)
 {
  restart:
     auto stepNr = allocBuildStep(txn, build->id);
@@ -457,6 +469,15 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
          res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt,
          isCachedBuild ? 1 : 0);
 
+    for (auto & [outputName, outputPath] : res.outputs) {
+        txn.exec_params0
+            ("update BuildOutputs set path = $3 where build = $1 and name = $2",
+             build->id,
+             outputName,
+             localStore->printStorePath(outputPath)
+            );
+    }
+
     txn.exec_params0("delete from BuildProducts where build = $1", build->id);
 
     unsigned int productNr = 1;
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 6c339af6..cee5b6cc 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,15 +192,14 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
-                        if (i.second.second) {
-                            auto res = txn.exec_params
-                                ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
-                                 localStore->printStorePath(*i.second.second));
-                            if (!res[0][0].is_null()) {
-                                propagatedFrom = res[0][0].as<BuildID>();
-                                break;
-                            }
+                    for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) {
+                          auto res = txn.exec_params
+                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
+                                localStore->printStorePath(ex.step->drvPath),
+                                i.first);
+                          if (!res[0][0].is_null()) {
+                              propagatedFrom = res[0][0].as<BuildID>();
+                              break;
                         }
                     }
                 }
@@ -236,12 +235,10 @@ bool State::getQueuedBuilds(Connection & conn,
         /* If we didn't get a step, it means the step's outputs are
            all valid. So we mark this as a finished, cached build. */
         if (!step) {
-            auto drv = localStore->readDerivation(build->drvPath);
-            BuildOutput res = getBuildOutputCached(conn, destStore, drv);
+            BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);
 
-            for (auto & i : drv.outputsAndOptPaths(*localStore))
-                if (i.second.second)
-                    addRoot(*i.second.second);
+            for (auto & i : localStore->queryDerivationOutputMap(build->drvPath))
+                addRoot(i.second);
 
             {
             auto mc = startDbUpdate();
@@ -481,26 +478,39 @@ Step::ptr State::createStep(ref<Store> destStore,
         throw PreviousFailure{step};
 
     /* Are all outputs valid? */
+    auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
     bool valid = true;
-    DerivationOutputs missing;
-    for (auto & i : step->drv->outputs)
-        if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
-            valid = false;
-            missing.insert_or_assign(i.first, i.second);
+    std::map<DrvOutput, std::optional<StorePath>> missing;
+    for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) {
+        auto outputHash = outputHashes.at(outputName);
+        if (maybeOutputPath.second) {
+            if (!destStore->isValidPath(*maybeOutputPath.second)) {
+                valid = false;
+                missing.insert({{outputHash, outputName}, maybeOutputPath.second});
+            }
+        } else {
+            experimentalFeatureSettings.require(Xp::CaDerivations);
+            if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) {
+                valid = false;
+                missing.insert({{outputHash, outputName}, std::nullopt});
+            }
         }
+    }
 
     /* Try to copy the missing paths from the local store or from
        substitutes. */
     if (!missing.empty()) {
 
         size_t avail = 0;
-        for (auto & i : missing) {
-            auto path = i.second.path(*localStore, step->drv->name, i.first);
-            if (/* localStore != destStore && */ localStore->isValidPath(*path))
+        for (auto & [i, maybePath] : missing) {
+            if ((maybePath && localStore->isValidPath(*maybePath)))
                 avail++;
-            else if (useSubstitutes) {
+            else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) {
+                maybePath = localStore->queryRealisation(i)->outPath;
+                avail++;
+            } else if (useSubstitutes && maybePath) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -508,44 +518,44 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         if (missing.size() == avail) {
             valid = true;
-            for (auto & i : missing) {
-                auto path = i.second.path(*localStore, step->drv->name, i.first);
+            for (auto & [i, path] : missing) {
+                if (path) {
+                    try {
+                        time_t startTime = time(0);
 
-                try {
-                    time_t startTime = time(0);
+                        if (localStore->isValidPath(*path))
+                            printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                                localStore->printStorePath(*path),
+                                localStore->printStorePath(drvPath));
+                        else {
+                            printInfo("substituting output ‘%1%’ of ‘%2%’",
+                                localStore->printStorePath(*path),
+                                localStore->printStorePath(drvPath));
+                            localStore->ensurePath(*path);
+                            // FIXME: should copy directly from substituter to destStore.
+                        }
 
-                    if (localStore->isValidPath(*path))
-                        printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                        StorePathSet closure;
+                        localStore->computeFSClosure({*path}, closure);
+                        copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
+
+                        time_t stopTime = time(0);
+
+                        {
+                            auto mc = startDbUpdate();
+                            pqxx::work txn(conn);
+                            createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path);
+                            txn.commit();
+                        }
+
+                    } catch (Error & e) {
+                        printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
                             localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath));
-                    else {
-                        printInfo("substituting output ‘%1%’ of ‘%2%’",
-                            localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath));
-                        localStore->ensurePath(*path);
-                        // FIXME: should copy directly from substituter to destStore.
+                            localStore->printStorePath(drvPath),
+                            e.what());
+                        valid = false;
+                        break;
                     }
-
-                    copyClosure(*localStore, *destStore,
-                        StorePathSet { *path },
-                        NoRepair, CheckSigs, NoSubstitute);
-
-                    time_t stopTime = time(0);
-
-                    {
-                        auto mc = startDbUpdate();
-                        pqxx::work txn(conn);
-                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
-                        txn.commit();
-                    }
-
-                } catch (Error & e) {
-                    printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                        localStore->printStorePath(*path),
-                        localStore->printStorePath(drvPath),
-                        e.what());
-                    valid = false;
-                    break;
                 }
             }
         }
@@ -640,17 +650,20 @@ void State::processJobsetSharesChange(Connection & conn)
 }
 
 
-BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::Derivation & drv)
+BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::StorePath & drvPath)
 {
+
+    auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath);
+
     {
     pqxx::work txn(conn);
 
-    for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
+    for (auto & [name, output] : derivationOutputs) {
         auto r = txn.exec_params
             ("select id, buildStatus, releaseName, closureSize, size from Builds b "
              "join BuildOutputs o on b.id = o.build "
              "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
-             localStore->printStorePath(*output.second));
+             localStore->printStorePath(output));
         if (r.empty()) continue;
         BuildID id = r[0][0].as<BuildID>();
 
@@ -704,5 +717,5 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
     }
 
     NarMemberDatas narMembers;
-    return getBuildOutput(destStore, narMembers, drv);
+    return getBuildOutput(destStore, narMembers, derivationOutputs);
 }
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 6359063a..5f01de1e 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -520,7 +520,7 @@ private:
         const std::string & machine);
 
     int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
-        Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath);
+        Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath);
 
     void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status);
 
@@ -536,7 +536,7 @@ private:
     void processQueueChange(Connection & conn);
 
     BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
-        const nix::Derivation & drv);
+        const nix::StorePath & drvPath);
 
     Step::ptr createStep(nix::ref<nix::Store> store,
         Connection & conn, Build::ptr build, const nix::StorePath & drvPath,

From 8046ec266851d8b8d3f08900cf9ba905ed344ed8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 16:21:56 -0500
Subject: [PATCH 02/10] Remove unused `outputHashes` variable

This looks like a stray copy paste.
---
 src/hydra-queue-runner/build-remote.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 26f6d63f..e26a8697 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -192,7 +192,6 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
 BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
 {
     BasicDerivation ret;
-    auto outputHashes = staticOutputHashes(store, drv);
     if (!drv.type().hasKnownOutputPaths()) {
         auto maybeBasicDrv = drv.tryResolve(store);
         if (!maybeBasicDrv)

From e3443cd22a3bc20c7d17f4e76d3868ea755b4b37 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 4 Dec 2023 17:41:11 -0500
Subject: [PATCH 03/10] Put back nicer `copyClosure` instead of manual closure
 + copy

It looks like we accidentally got the old code back, probably after a merge
conflict resolution.
---
 src/hydra-queue-runner/queue-monitor.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index cee5b6cc..04917932 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -535,9 +535,9 @@ Step::ptr State::createStep(ref<Store> destStore,
                             // FIXME: should copy directly from substituter to destStore.
                         }
 
-                        StorePathSet closure;
-                        localStore->computeFSClosure({*path}, closure);
-                        copyPaths(*localStore, *destStore, closure, NoRepair, CheckSigs, NoSubstitute);
+                        copyClosure(*localStore, *destStore,
+                            StorePathSet { *path },
+                            NoRepair, CheckSigs, NoSubstitute);
 
                         time_t stopTime = time(0);
 

From 069b7775c565f5999fe33e8c3f28c7b9306039ca Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 11:26:26 -0500
Subject: [PATCH 04/10] hydra-eval-jobs: Ensure we have output path if
 ca-derivations is disabled

Brought up by @thufschmitt in
https://github.com/NixOS/hydra/pull/1316#discussion_r1415111329 . This
makes this closer to what was originally there --- which just dispatched
off the experimental feature rather than the presence/absense of the
output, too.
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index d007189d..b44ccbb3 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -243,11 +243,16 @@ static void worker(
                 }
 
                 nlohmann::json out;
-                for (auto & [outputName, optOutputPath] : outputs)
-                    if (optOutputPath)
+                for (auto & [outputName, optOutputPath] : outputs) {
+                    if (optOutputPath) {
                         out[outputName] = state.store->printStorePath(*optOutputPath);
-                    else
+                    } else {
+                        // See the `queryOutputs` call above; we should
+                        // not encounter missing output paths otherwise.
+                        assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
                         out[outputName] = "";
+                    }
+                }
                 job["outputs"] = std::move(out);
                 reply["job"] = std::move(job);
             }

From 3df8feb3a2b30b733b141f6e6e43572998311524 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 11:27:52 -0500
Subject: [PATCH 05/10] Add TODO about setting `null` instead of empty string
 in JSON

An empty string is a sneaky way to avoid hard failures --- things that
expect strings still get strings, but it does conversely open the door
up to soft failures (spooky-action-at-a-distance ones because the string
did not have the expected invariants).

"Fail fast" with null will ultimately make the system more robust, but
force us to fix more things up front, and I don't want to change this
without also fixing those things up front, especially as this commit is
for now just part of the the preparatory PR for which this is dead code.
---
 src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
index b44ccbb3..2794cc62 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
@@ -250,6 +250,10 @@ static void worker(
                         // See the `queryOutputs` call above; we should
                         // not encounter missing output paths otherwise.
                         assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
+                        // TODO it would be better to set `null` than an
+                        // empty string here, to force the consumer of
+                        // this JSON to more explicitly handle this
+                        // case.
                         out[outputName] = "";
                     }
                 }

From 11f8030b0f4c75ed7640563d68200cf3ec59edec Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 6 Dec 2023 17:59:25 -0500
Subject: [PATCH 06/10] Add comment from GitHub about adding to store as code
 comment

---
 src/hydra-queue-runner/build-remote.cc | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index e26a8697..94a01676 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -425,6 +425,19 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
+    // Why both stores? @thufschmitt says:
+    //
+    // > I think it's an easy (and terribly inefficient 😬) way of
+    // making sure that `localStore.queryRealisations` will succeed
+    // (which we IIRC we need later to get back some metadata about the
+    // path to put it in the db).
+    // >
+    // > To be honest, we shouldn't do that but instead carry the needed
+    // metadata in memory until the point where we need it (but that can
+    // come later once we're confident that this is at least correct)
+    //
+    // TODO make the above change to avoid copying excess data back and
+    // forth.
     for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the

From 2ee0068fdca7d9fb8bef23b3671a86b8f0fcff07 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 15:05:03 -0500
Subject: [PATCH 07/10] Do not copy for both stores for now

It has a performance cost, and as the comment says we should be doing
the better solution. We want to land this preparatory change on prod
while the rest is still on staging, so we should just skip it for now.

Skipping it will not affect regular fixed-output and input-addressed
derivations, which are the only ones prod would deal with upon getting
this code.

The main CA derivations support branch will revert this commit so it
still works.
---
 src/hydra-queue-runner/build-remote.cc | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 1d0bcc35..77484244 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -425,20 +425,6 @@ static void copyPathFromRemote(
     const ValidPathInfo & info
 )
 {
-    // Why both stores? @thufschmitt says:
-    //
-    // > I think it's an easy (and terribly inefficient 😬) way of
-    // making sure that `localStore.queryRealisations` will succeed
-    // (which we IIRC we need later to get back some metadata about the
-    // path to put it in the db).
-    // >
-    // > To be honest, we shouldn't do that but instead carry the needed
-    // metadata in memory until the point where we need it (but that can
-    // come later once we're confident that this is at least correct)
-    //
-    // TODO make the above change to avoid copying excess data back and
-    // forth.
-    for (auto * store : {&destStore, &localStore}) {
       /* Receive the NAR from the remote and add it to the
           destination store. Meanwhile, extract all the info from the
           NAR that getBuildOutput() needs. */
@@ -458,8 +444,7 @@ static void copyPathFromRemote(
           extractNarData(tee, localStore.printStorePath(info.path), narMembers);
       });
 
-      store->addToStore(info, *source2, NoRepair, NoCheckSigs);
-    }
+      destStore.addToStore(info, *source2, NoRepair, NoCheckSigs);
 }
 
 static void copyPathsFromRemote(

From ebfefb9161046c83bbbebd26f4cb37ac69628b40 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 12:46:36 -0500
Subject: [PATCH 08/10] Sync up with some changes done to the main CA branch

---
 flake.lock                                   |   8 +-
 flake.nix                                    |   2 +-
 src/hydra-queue-runner/build-remote.cc       |  55 +++------
 src/hydra-queue-runner/builder.cc            |  11 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |   4 +-
 src/hydra-queue-runner/queue-monitor.cc      | 119 ++++++++++---------
 6 files changed, 94 insertions(+), 105 deletions(-)

diff --git a/flake.lock b/flake.lock
index 2871e70a..e7f9224c 100644
--- a/flake.lock
+++ b/flake.lock
@@ -42,16 +42,16 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1701122567,
-        "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=",
+        "lastModified": 1702314838,
+        "narHash": "sha256-calxK+fZ4/tZy1fbph8qyx4ePUAf4ZdvIugpzWeFIGE=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d",
+        "rev": "ae451e2247b18be6bd36b9d85e41b632e774f40b",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.19.2",
+        "ref": "2.19-maintenance",
         "repo": "nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 5a5aef55..8280e076 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.19.2";
+  inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 377bdb56..65f9f3fb 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -182,40 +182,6 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
     return sorted;
 }
 
-/**
- * Replace the input derivations by their output paths to send a minimal closure
- * to the builder.
- *
- * If we can afford it, resolve it, so that the newly generated derivation still
- * has some sensible output paths.
- */
-BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
-{
-    BasicDerivation ret;
-    if (!drv.type().hasKnownOutputPaths()) {
-        auto maybeBasicDrv = drv.tryResolve(store);
-        if (!maybeBasicDrv)
-            throw Error(
-                "the derivation '%s' can’t be resolved. It’s probably "
-                "missing some outputs",
-                store.printStorePath(drvPath));
-        ret = *maybeBasicDrv;
-    } else {
-        // If the derivation is a real `InputAddressed` derivation, we must
-        // resolve it manually to keep the original output paths
-        ret = BasicDerivation(drv);
-        for (auto & [drvPath, node] : drv.inputDrvs.map) {
-            auto drv2 = store.readDerivation(drvPath);
-            auto drv2Outputs = drv2.outputsAndOptPaths(store);
-            for (auto & name : node.value) {
-                auto inputPath = drv2Outputs.at(name);
-                ret.inputSrcs.insert(*inputPath.second);
-            }
-        }
-    }
-    return ret;
-}
-
 static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
 {
     std::string base(drvPath.to_string());
@@ -262,7 +228,22 @@ static BasicDerivation sendInputs(
     counter & nrStepsCopyingTo
 )
 {
-    BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath);
+    /* Replace the input derivations by their output paths to send a
+       minimal closure to the builder.
+
+       `tryResolve` currently does *not* rewrite input addresses, so it
+       is safe to do this in all cases. (It should probably have a mode
+       to do that, however, but we would not use it here.)
+     */
+    BasicDerivation basicDrv = ({
+        auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore);
+        if (!maybeBasicDrv)
+            throw Error(
+                "the derivation '%s' can’t be resolved. It’s probably "
+                "missing some outputs",
+                localStore.printStorePath(step.drvPath));
+        *maybeBasicDrv;
+    });
 
     /* Ensure that the inputs exist in the destination store. This is
        a no-op for regular stores, but for the binary cache store,
@@ -351,6 +332,8 @@ static BuildResult performBuild(
         // far anyways
         assert(drv.type().hasKnownOutputPaths());
         DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore);
+        // Since this a `BasicDerivation`, `staticOutputHashes` will not
+        // do any real work.
         auto outputHashes = staticOutputHashes(localStore, drv);
         for (auto & [outputName, output] : drvOutputs) {
             auto outputPath = output.second;
@@ -665,14 +648,12 @@ void State::buildRemote(ref<Store> destStore,
             auto outputHashes = staticOutputHashes(*localStore, *step->drv);
             for (auto & [outputName, realisation] : buildResult.builtOutputs) {
                 // Register the resolved drv output
-                localStore->registerDrvOutput(realisation);
                 destStore->registerDrvOutput(realisation);
 
                 // Also register the unresolved one
                 auto unresolvedRealisation = realisation;
                 unresolvedRealisation.signatures.clear();
                 unresolvedRealisation.id.drvHash = outputHashes.at(outputName);
-                localStore->registerDrvOutput(unresolvedRealisation);
                 destStore->registerDrvOutput(unresolvedRealisation);
             }
         }
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 42983941..b1b58c05 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         if (result.stepStatus == bsSuccess) {
             updateStep(ssPostProcessing);
-            res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath));
+            res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore));
         }
     }
 
@@ -277,9 +277,12 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
 
         assert(stepNr);
 
-        for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) {
-            if (i.second)
-               addRoot(*i.second);
+        for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) {
+            if (!optOutputPath)
+                throw Error(
+                    "Missing output %s for derivation %d which was supposed to have succeeded",
+                    outputName, localStore->printStorePath(step->drvPath));
+            addRoot(*optOutputPath);
         }
 
         /* Register success in the database for all Build objects that
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 2f2c6091..d514ecfa 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -312,7 +312,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
 
     if (r.affected_rows() == 0) goto restart;
 
-    for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath))
+    for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
             buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
@@ -359,7 +359,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
         assert(res.size());
         StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
         // If we've finished building, all the paths should be known
-        for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath))
+        for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore))
             txn.exec_params0
                 ("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
                   buildId, stepNr, name, localStore->printStorePath(output));
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index 04917932..a4d9b4dc 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,11 +192,11 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) {
+                    for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
                           auto res = txn.exec_params
                               ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
                                 localStore->printStorePath(ex.step->drvPath),
-                                i.first);
+                                outputName);
                           if (!res[0][0].is_null()) {
                               propagatedFrom = res[0][0].as<BuildID>();
                               break;
@@ -237,7 +237,7 @@ bool State::getQueuedBuilds(Connection & conn,
         if (!step) {
             BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);
 
-            for (auto & i : localStore->queryDerivationOutputMap(build->drvPath))
+            for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore))
                 addRoot(i.second);
 
             {
@@ -481,20 +481,12 @@ Step::ptr State::createStep(ref<Store> destStore,
     auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
     bool valid = true;
     std::map<DrvOutput, std::optional<StorePath>> missing;
-    for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) {
+    for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
         auto outputHash = outputHashes.at(outputName);
-        if (maybeOutputPath.second) {
-            if (!destStore->isValidPath(*maybeOutputPath.second)) {
-                valid = false;
-                missing.insert({{outputHash, outputName}, maybeOutputPath.second});
-            }
-        } else {
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-            if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) {
-                valid = false;
-                missing.insert({{outputHash, outputName}, std::nullopt});
-            }
-        }
+        if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
+            continue;
+        valid = false;
+        missing.insert({{outputHash, outputName}, maybeOutputPath});
     }
 
     /* Try to copy the missing paths from the local store or from
@@ -503,14 +495,24 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         size_t avail = 0;
         for (auto & [i, maybePath] : missing) {
-            if ((maybePath && localStore->isValidPath(*maybePath)))
+            // If we don't know the output path from the destination
+            // store, see if the local store can tell us.
+            if (/* localStore != destStore && */ !maybePath && experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
+                if (auto maybeRealisation = localStore->queryRealisation(i))
+                    maybePath = maybeRealisation->outPath;
+
+            if (!maybePath) {
+                // No hope of getting the store object if we don't know
+                // the path.
+                continue;
+            }
+            auto & path = *maybePath;
+
+            if (/* localStore != destStore && */ localStore->isValidPath(path))
                 avail++;
-            else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) {
-                maybePath = localStore->queryRealisation(i)->outPath;
-                avail++;
-            } else if (useSubstitutes && maybePath) {
+            else if (useSubstitutes) {
                 SubstitutablePathInfos infos;
-                localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos);
+                localStore->querySubstitutablePathInfos({{path, {}}}, infos);
                 if (infos.size() == 1)
                     avail++;
             }
@@ -518,44 +520,47 @@ Step::ptr State::createStep(ref<Store> destStore,
 
         if (missing.size() == avail) {
             valid = true;
-            for (auto & [i, path] : missing) {
-                if (path) {
-                    try {
-                        time_t startTime = time(0);
+            for (auto & [i, maybePath] : missing) {
+                // If we found everything, then we should know the path
+                // to every missing store object now.
+                assert(maybePath);
+                auto & path = *maybePath;
 
-                        if (localStore->isValidPath(*path))
-                            printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
-                                localStore->printStorePath(*path),
-                                localStore->printStorePath(drvPath));
-                        else {
-                            printInfo("substituting output ‘%1%’ of ‘%2%’",
-                                localStore->printStorePath(*path),
-                                localStore->printStorePath(drvPath));
-                            localStore->ensurePath(*path);
-                            // FIXME: should copy directly from substituter to destStore.
-                        }
+                try {
+                    time_t startTime = time(0);
 
-                        copyClosure(*localStore, *destStore,
-                            StorePathSet { *path },
-                            NoRepair, CheckSigs, NoSubstitute);
-
-                        time_t stopTime = time(0);
-
-                        {
-                            auto mc = startDbUpdate();
-                            pqxx::work txn(conn);
-                            createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path);
-                            txn.commit();
-                        }
-
-                    } catch (Error & e) {
-                        printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
-                            localStore->printStorePath(*path),
-                            localStore->printStorePath(drvPath),
-                            e.what());
-                        valid = false;
-                        break;
+                    if (localStore->isValidPath(path))
+                        printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
+                            localStore->printStorePath(path),
+                            localStore->printStorePath(drvPath));
+                    else {
+                        printInfo("substituting output ‘%1%’ of ‘%2%’",
+                            localStore->printStorePath(path),
+                            localStore->printStorePath(drvPath));
+                        localStore->ensurePath(path);
+                        // FIXME: should copy directly from substituter to destStore.
                     }
+
+                    copyClosure(*localStore, *destStore,
+                        StorePathSet { path },
+                        NoRepair, CheckSigs, NoSubstitute);
+
+                    time_t stopTime = time(0);
+
+                    {
+                        auto mc = startDbUpdate();
+                        pqxx::work txn(conn);
+                        createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path);
+                        txn.commit();
+                    }
+
+                } catch (Error & e) {
+                    printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
+                        localStore->printStorePath(path),
+                        localStore->printStorePath(drvPath),
+                        e.what());
+                    valid = false;
+                    break;
                 }
             }
         }

From a6b6c5a5392a7e705825e8dac7cb818a1f9944fe Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 12:55:57 -0500
Subject: [PATCH 09/10] Revert query -- those columns don't exist yet!

---
 src/hydra-queue-runner/queue-monitor.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index a4d9b4dc..fc344244 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -192,11 +192,12 @@ bool State::getQueuedBuilds(Connection & conn,
                 if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
 
                 if (!propagatedFrom) {
-                    for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
+                    for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
+                          // ca-derivations not actually supported yet
+                          assert(optOutputPath);
                           auto res = txn.exec_params
-                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
-                                localStore->printStorePath(ex.step->drvPath),
-                                outputName);
+                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
+                               localStore->printStorePath(*optOutputPath));
                           if (!res[0][0].is_null()) {
                               propagatedFrom = res[0][0].as<BuildID>();
                               break;

From 6e67884ff164d9d1c115aef066e18f8de0c5fc44 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 14:05:18 -0500
Subject: [PATCH 10/10] One more `queryDerivationOutputMap` should use the eval
 store param

---
 src/hydra-queue-runner/queue-monitor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index fc344244..81678fe1 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -658,8 +658,7 @@ void State::processJobsetSharesChange(Connection & conn)
 
 BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, const nix::StorePath & drvPath)
 {
-
-    auto derivationOutputs = localStore->queryDerivationOutputMap(drvPath);
+    auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore);
 
     {
     pqxx::work txn(conn);