Merge pull request #8829 from obsidiansystems/build-dynamic-derivations

Adapt scheduler to work with dynamic derivations

src/libexpr/primops.cc

@@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V
 {
     SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
 
-    std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+    OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
 
     state.mkSingleDerivedPathString(
         SingleDerivedPath::Built {

src/libstore/build/create-derivation-and-realise-goal.cc

@@ -0,0 +1,157 @@
+#include "create-derivation-and-realise-goal.hh"
+#include "worker.hh"
+
+namespace nix {
+
+CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
+    : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
+    , drvReq(drvReq)
+    , wantedOutputs(wantedOutputs)
+    , buildMode(buildMode)
+{
+    state = &CreateDerivationAndRealiseGoal::getDerivation;
+    name = fmt(
+        "outer obtaining drv from '%s' and then building outputs %s",
+        drvReq->to_string(worker.store),
+        std::visit(overloaded {
+            [&](const OutputsSpec::All) -> std::string {
+                return "* (all of them)";
+            },
+            [&](const OutputsSpec::Names os) {
+                return concatStringsSep(", ", quoteStrings(os));
+            },
+        }, wantedOutputs.raw));
+    trace("created outer");
+
+    worker.updateProgress();
+}
+
+
+CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
+{
+}
+
+
+static StorePath pathPartOfReq(const SingleDerivedPath & req)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+        [&](const SingleDerivedPath::Built & bfd) {
+            return pathPartOfReq(*bfd.drvPath);
+        },
+    }, req.raw());
+}
+
+
+std::string CreateDerivationAndRealiseGoal::key()
+{
+    /* Ensure that derivations get built in order of their name,
+       i.e. a derivation named "aardvark" always comes before "baboon". And
+       substitution goals and inner derivation goals always happen before
+       derivation goals (due to "b$"). */
+    return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
+}
+
+
+void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
+{
+}
+
+
+void CreateDerivationAndRealiseGoal::work()
+{
+    (this->*state)();
+}
+
+
+void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
+{
+    /* If we already want all outputs, there is nothing to do. */
+    auto newWanted = wantedOutputs.union_(outputs);
+    bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
+    wantedOutputs = newWanted;
+
+    if (!needRestart) return;
+
+    if (!optDrvPath)
+        // haven't started steps where the outputs matter yet
+        return;
+    worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
+}
+
+
+void CreateDerivationAndRealiseGoal::getDerivation()
+{
+    trace("outer init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (auto optDrvPath = [this]() -> std::optional<StorePath> {
+        if (buildMode != bmNormal) return std::nullopt;
+
+        auto drvPath = StorePath::dummy;
+        try {
+            drvPath = resolveDerivedPath(worker.store, *drvReq);
+        } catch (MissingRealisation) {
+            return std::nullopt;
+        }
+        return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
+            ? std::optional { drvPath }
+            : std::nullopt;
+    }()) {
+        trace(fmt("already have drv '%s' for '%s', can go straight to building",
+            worker.store.printStorePath(*optDrvPath),
+            drvReq->to_string(worker.store)));
+
+        loadAndBuildDerivation();
+    } else {
+        trace("need to obtain drv we want to build");
+
+        addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
+
+        state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
+        if (waitees.empty()) work();
+    }
+}
+
+
+void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
+{
+    trace("outer load and build derivation");
+
+    if (nrFailed != 0) {
+        amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
+        return;
+    }
+
+    StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
+    /* Build this step! */
+    concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
+    addWaitee(upcast_goal(concreteDrvGoal));
+    state = &CreateDerivationAndRealiseGoal::buildDone;
+    optDrvPath = std::move(drvPath);
+    if (waitees.empty()) work();
+}
+
+
+void CreateDerivationAndRealiseGoal::buildDone()
+{
+    trace("outer build done");
+
+    buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
+        .drvPath = drvReq,
+        .outputs = wantedOutputs,
+    });
+
+    if (buildResult.success())
+        amDone(ecSuccess);
+    else
+        amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
+}
+
+
+}

src/libstore/build/create-derivation-and-realise-goal.hh

@@ -0,0 +1,96 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
+#include "goal.hh"
+
+namespace nix {
+
+struct DerivationGoal;
+
+/**
+ * This goal type is essentially the serial composition (like function
+ * composition) of a goal for getting a derivation, and then a
+ * `DerivationGoal` using the newly-obtained derivation.
+ *
+ * In the (currently experimental) general inductive case of derivations
+ * that are themselves build outputs, that first goal will be *another*
+ * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
+ * where the derivation has no provence and is just referred to by
+ * (content-addressed) store path, that first goal is a
+ * `SubstitutionGoal`.
+ *
+ * If we already have the derivation (e.g. if the evalutator has created
+ * the derivation locally and then instructured the store to build it),
+ * we can skip the first goal entirely as a small optimization.
+ */
+struct CreateDerivationAndRealiseGoal : public Goal
+{
+    /**
+     * How to obtain a store path of the derivation to build.
+     */
+    ref<SingleDerivedPath> drvReq;
+
+    /**
+     * The path of the derivation, once obtained.
+     **/
+    std::optional<StorePath> optDrvPath;
+
+    /**
+     * The goal for the corresponding concrete derivation.
+     **/
+    std::shared_ptr<DerivationGoal> concreteDrvGoal;
+
+    /**
+     * The specific outputs that we need to build.
+     */
+    OutputsSpec wantedOutputs;
+
+    typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
+    GoalState state;
+
+    /**
+     * The final output paths of the build.
+     *
+     * - For input-addressed derivations, always the precomputed paths
+     *
+     * - For content-addressed derivations, calcuated from whatever the
+     *   hash ends up being. (Note that fixed outputs derivations that
+     *   produce the "wrong" output still install that data under its
+     *   true content-address.)
+     */
+    OutputPathMap finalOutputs;
+
+    BuildMode buildMode;
+
+    CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+        const OutputsSpec & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
+    virtual ~CreateDerivationAndRealiseGoal();
+
+    void timedOut(Error && ex) override;
+
+    std::string key() override;
+
+    void work() override;
+
+    /**
+     * Add wanted outputs to an already existing derivation goal.
+     */
+    void addWantedOutputs(const OutputsSpec & outputs);
+
+    /**
+     * The states.
+     */
+    void getDerivation();
+    void loadAndBuildDerivation();
+    void buildDone();
+
+    JobCategory jobCategory() const override {
+        return JobCategory::Administration;
+    };
+};
+
+}

src/libstore/build/derivation-goal.cc

@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::getDerivation;
+    state = &DerivationGoal::loadDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
 void DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
@@ -380,7 +362,12 @@ void DerivationGoal::gaveUpOnSubstitution()
                         worker.store.printStorePath(i.first));
             }
 
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(i.first),
+                    .outputs = i.second,
+                },
+                buildMode == bmRepair ? bmRepair : bmNormal));
         }
 
     /* Copy the input sources from the eval store to the build
@@ -452,7 +439,12 @@ void DerivationGoal::repairClosure()
         if (drvPath2 == outputsToDrv.end())
             addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(drvPath2->second),
+                    .outputs = OutputsSpec::All { },
+                },
+                bmRepair));
     }
 
     if (waitees.empty()) {
@@ -1483,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
+    auto * dg = tryGetConcreteDrvGoal(waitee);
     if (!dg) return;
 
     auto outputs = fullDrv.inputDrvs.find(dg->drvPath);

src/libstore/build/derivation-goal.hh

@@ -50,6 +50,13 @@ struct InitialOutput {
     std::optional<InitialOutputStatus> known;
 };
 
+/**
+ * A goal for building some or all of the outputs of a derivation.
+ *
+ * The derivation must already be present, either in the store in a drv
+ * or in memory. If the derivation itself needs to be gotten first, a
+ * `CreateDerivationAndRealiseGoal` goal must be used instead.
+ */
 struct DerivationGoal : public Goal
 {
     /**
@@ -66,8 +73,7 @@ struct DerivationGoal : public Goal
     std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
     /**
-     * The specific outputs that we need to build.  Empty means all of
+     * The specific outputs that we need to build.
-     * them.
      */
     OutputsSpec wantedOutputs;
 
@@ -229,7 +235,6 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
-    void getDerivation();
     void loadDerivation();
     void haveDerivation();
     void outputsSubstitutionTried();
@@ -334,7 +339,9 @@ struct DerivationGoal : public Goal
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
 
-    JobCategory jobCategory() override { return JobCategory::Build; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Build;
+    };
 };
 
 MakeError(NotDeterministic, BuildError);

src/libstore/build/drv-output-substitution-goal.hh

@@ -73,7 +73,9 @@ public:
     void work() override;
     void handleEOF(int fd) override;
 
-    JobCategory jobCategory() override { return JobCategory::Substitution; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
 };
 
 }

src/libstore/build/entry-points.cc

@@ -1,5 +1,6 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
 #include "derivation-goal.hh"
 #include "local-store.hh"
 
@@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StorePathSet failed;
+    StringSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
                 ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
-            if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
+            if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
-            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+                failed.insert(i2->drvReq->to_string(*this));
+            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
+                failed.insert(printStorePath(i2->storePath));
         }
     }
 
@@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
+        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
     }
 }
 
@@ -124,8 +127,11 @@ void Store::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             goals.clear();
+            goals.insert(worker.makeGoal(DerivedPath::Built {
+                .drvPath = makeConstantStorePathRef(*info->deriver),
                 // FIXME: Should just build the specific output we need.
-            goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
+                .outputs = OutputsSpec::All { },
+            }, bmRepair));
             worker.run(goals);
         } else
             throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));

src/libstore/build/goal.cc

@@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
 }
 
 
-BuildResult Goal::getBuildResult(const DerivedPath & req) {
+BuildResult Goal::getBuildResult(const DerivedPath & req) const {
     BuildResult res { buildResult };
 
     if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {

src/libstore/build/goal.hh

@@ -41,8 +41,24 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
  * of each category in parallel.
  */
 enum struct JobCategory {
+    /**
+     * A build of a derivation; it will use CPU and disk resources.
+     */
     Build,
+    /**
+     * A substitution an arbitrary store object; it will use network resources.
+     */
     Substitution,
+    /**
+     * A goal that does no "real" work by itself, and just exists to depend on
+     * other goals which *do* do real work. These goals therefore are not
+     * limited.
+     *
+     * These goals cannot infinitely create themselves, so there is no risk of
+     * a "fork bomb" type situation (which would be a problem even though the
+     * goal do no real work) either.
+     */
+    Administration,
 };
 
 struct Goal : public std::enable_shared_from_this<Goal>
@@ -110,7 +126,7 @@ public:
      * sake of both privacy and determinism, and this "safe accessor"
      * ensures we don't.
      */
-    BuildResult getBuildResult(const DerivedPath &);
+    BuildResult getBuildResult(const DerivedPath &) const;
 
     /**
      * Exception containing an error message, if any.
@@ -144,7 +160,7 @@ public:
 
     void trace(std::string_view s);
 
-    std::string getName()
+    std::string getName() const
     {
         return name;
     }
@@ -166,7 +182,7 @@ public:
      * @brief Hint for the scheduler, which concurrency limit applies.
      * @see JobCategory
      */
-    virtual JobCategory jobCategory() = 0;
+    virtual JobCategory jobCategory() const = 0;
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);

src/libstore/build/local-derivation-goal.cc

@@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd)
 }
 
 
-StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
+StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),

src/libstore/build/local-derivation-goal.hh

@@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal
      * @todo Add option to randomize, so we can audit whether our
      * rewrites caught everything
      */
-    StorePath makeFallbackPath(std::string_view outputName);
+    StorePath makeFallbackPath(OutputNameView outputName);
 };
 
 }

src/libstore/build/substitution-goal.hh

@@ -117,7 +117,9 @@ public:
     /* Called by destructor, can't be overridden */
     void cleanup() override final;
 
-    JobCategory jobCategory() override { return JobCategory::Substitution; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
 };
 
 }

src/libstore/build/worker.cc

@@ -2,6 +2,7 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "drv-output-substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
@@ -41,6 +42,24 @@ Worker::~Worker()
 }
 
 
+std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
+    ref<SingleDerivedPath> drvReq,
+    const OutputsSpec & wantedOutputs,
+    BuildMode buildMode)
+{
+    std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
+    std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
+    if (!goal) {
+        goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
+        goal_weak = goal;
+        wakeUp(goal);
+    } else {
+        goal->addWantedOutputs(wantedOutputs);
+    }
+    return goal;
+}
+
+
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
@@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & bfd) -> GoalPtr {
-            if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
+            return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
-                return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
-            else
-                throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
         [&](const DerivedPath::Opaque & bo) -> GoalPtr {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 }
 
 
+template<typename K, typename V, typename F>
+static void cullMap(std::map<K, V> & goalMap, F f)
+{
+    for (auto i = goalMap.begin(); i != goalMap.end();)
+        if (!f(i->second))
+            i = goalMap.erase(i);
+        else ++i;
+}
+
+
 template<typename K, typename G>
 static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    for (auto i = goalMap.begin();
+    cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
-         i != goalMap.end(); )
+        return gp.lock() != goal;
-        if (i->second.lock() == goal) {
+    });
-            auto j = i; ++j;
-            goalMap.erase(i);
-            i = j;
 }
-        else ++i;
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
+{
+    /* !!! inefficient */
+    cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
+        if (node.value.lock() == goal)
+            node.value.reset();
+        removeGoal(goal, node.childMap);
+        return !node.value.expired() || !node.childMap.empty();
+    });
 }
 
 
 void Worker::removeGoal(GoalPtr goal)
 {
-    if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+    if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
+        nix::removeGoal(drvGoal, outerDerivationGoals.map);
+    else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
@@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
+        switch (goal->jobCategory()) {
-        else nrLocalBuilds++;
+        case JobCategory::Substitution:
+            nrSubstitutions++;
+            break;
+        case JobCategory::Build:
+            nrLocalBuilds++;
+            break;
+        case JobCategory::Administration:
+            /* Intentionally not limited, see docs */
+            break;
+        default:
+            abort();
+        }
     }
 }
 
@@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) {
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-        } else {
+            break;
+        case JobCategory::Build:
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
+            break;
+        case JobCategory::Administration:
+            /* Intentionally not limited, see docs */
+            break;
+        default:
+            abort();
         }
     }
 
@@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals)
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
-        if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
+        if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Built {
-                .drvPath = makeConstantStorePathRef(goal->drvPath),
+                .drvPath = goal->drvReq,
                 .outputs = goal->wantedOutputs,
             });
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@@ -522,11 +579,26 @@ void Worker::markContentsGood(const StorePath & path)
 }
 
 
-GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal)
-    return subGoal;
+{
-}
-GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
     return subGoal;
 }
 
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
+{
+    return subGoal;
+}
+
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
+{
+    return subGoal;
+}
+
+const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
+{
+    auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
+    if (!odg) return nullptr;
+    return &*odg->concreteDrvGoal;
+}
+
 }

src/libstore/build/worker.hh

@@ -4,6 +4,7 @@
 #include "types.hh"
 #include "lock.hh"
 #include "store-api.hh"
+#include "derived-path-map.hh"
 #include "goal.hh"
 #include "realisation.hh"
 
@@ -13,6 +14,7 @@
 namespace nix {
 
 /* Forward definition. */
+struct CreateDerivationAndRealiseGoal;
 struct DerivationGoal;
 struct PathSubstitutionGoal;
 class DrvOutputSubstitutionGoal;
@@ -31,9 +33,23 @@ class DrvOutputSubstitutionGoal;
  */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
+/**
+ * The current implementation of impure derivations has
+ * `DerivationGoal`s accumulate realisations from their waitees.
+ * Unfortunately, `DerivationGoal`s don't directly depend on other
+ * goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
+ *
+ * We try not to share any of the details of any goal type with any
+ * other, for sake of modularity and quicker rebuilds. This means we
+ * cannot "just" downcast and fish out the field. So as an escape hatch,
+ * we have made the function, written in `worker.cc` where all the goal
+ * types are visible, and use it instead.
+ */
+const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
 
 /**
  * A mapping used to remember for each child process to what goal it
@@ -102,6 +118,9 @@ private:
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
+
+    DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
+
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
     std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@@ -189,6 +208,9 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
+    std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
+        ref<SingleDerivedPath> drvPath,
+        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

src/libstore/derivations.cc

@@ -12,7 +12,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     return std::visit(overloaded {
         [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -36,7 +36,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
@@ -466,7 +466,7 @@ bool isDerivation(std::string_view fileName)
 }
 
 
-std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+std::string outputPathName(std::string_view drvName, OutputNameView outputName) {
     std::string res { drvName };
     if (outputName != "out") {
         res += "-";
@@ -810,7 +810,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 }
 
 
-std::string hashPlaceholder(const std::string_view outputName)
+std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
     return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
@@ -963,7 +963,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
-    const Store & store, std::string_view drvName, std::string_view outputName) const
+    const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     nlohmann::json res = nlohmann::json::object();
     std::visit(overloaded {
@@ -990,7 +990,7 @@ nlohmann::json DerivationOutput::toJSON(
 
 
 DerivationOutput DerivationOutput::fromJSON(
-    const Store & store, std::string_view drvName, std::string_view outputName,
+    const Store & store, std::string_view drvName, OutputNameView outputName,
     const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {

src/libstore/derivations.hh

@@ -55,7 +55,7 @@ struct DerivationOutput
          * @param drvName The name of the derivation this is an output of, without the `.drv`.
          * @param outputName The name of this output.
          */
-        StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+        StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
 
         GENERATE_CMP(CAFixed, me->ca);
     };
@@ -132,19 +132,19 @@ struct DerivationOutput
      * the safer interface provided by
      * BasicDerivation::outputsAndOptPaths
      */
-    std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+    std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
 
     nlohmann::json toJSON(
         const Store & store,
         std::string_view drvName,
-        std::string_view outputName) const;
+        OutputNameView outputName) const;
     /**
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivationOutput fromJSON(
         const Store & store,
         std::string_view drvName,
-        std::string_view outputName,
+        OutputNameView outputName,
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
@@ -405,7 +405,7 @@ bool isDerivation(std::string_view fileName);
  * This is usually <drv-name>-<output-name>, but is just <drv-name> when
  * the output name is "out".
  */
-std::string outputPathName(std::string_view drvName, std::string_view outputName);
+std::string outputPathName(std::string_view drvName, OutputNameView outputName);
 
 
 /**
@@ -499,7 +499,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
  * own outputs without needing to use the hash of a derivation in
  * itself, making the hash near-impossible to calculate.
  */
-std::string hashPlaceholder(const std::string_view outputName);
+std::string hashPlaceholder(const OutputNameView outputName);
 
 extern const Hash impureOutputHash;
 

src/libstore/derived-path-map.cc

@@ -0,0 +1,33 @@
+#include "derived-path-map.hh"
+
+namespace nix {
+
+template<typename V>
+typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const SingleDerivedPath & k)
+{
+    std::function<ChildNode &(const SingleDerivedPath & )> initIter;
+    initIter = [&](const auto & k) -> auto & {
+        return std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & bo) -> auto & {
+                // will not overwrite if already there
+                return map[bo.path];
+            },
+            [&](const SingleDerivedPath::Built & bfd) -> auto & {
+                auto & n = initIter(*bfd.drvPath);
+                return n.childMap[bfd.output];
+            },
+        }, k.raw());
+    };
+    return initIter(k);
+}
+
+}
+
+// instantiations
+
+#include "create-derivation-and-realise-goal.hh"
+namespace nix {
+
+template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
+
+}

src/libstore/derived-path-map.hh

@@ -0,0 +1,73 @@
+#pragma once
+
+#include "types.hh"
+#include "derived-path.hh"
+
+namespace nix {
+
+/**
+ * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to
+ * values.
+ *
+ * Concretely, an n-ary tree, as described below. A
+ * `SingleDerivedPath::Opaque` maps to the value of an immediate child
+ * of the root node. A `SingleDerivedPath::Built` maps to a deeper child
+ * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a
+ * child node (inductively), and then the
+ * `SingleDerivedPath::Built::output` is used to look up that child's
+ * child via its map. In this manner, every `SingleDerivedPath` is
+ * mapped to a child node.
+ *
+ * @param V A type to instantiate for each output. It should probably
+ * should be an "optional" type so not every interior node has to have a
+ * value. For example, the scheduler uses
+ * `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
+ * remember which goals correspond to which outputs. `* const Something`
+ * or `std::optional<Something>` would also be good choices for
+ * "optional" types.
+ */
+template<typename V>
+struct DerivedPathMap {
+    /**
+     * A child node (non-root node).
+     */
+    struct ChildNode {
+        /**
+         * Value of this child node.
+         *
+         * @see DerivedPathMap for what `V` should be.
+         */
+        V value;
+
+        /**
+         * The map type for the root node.
+         */
+        using Map = std::map<OutputName, ChildNode>;
+
+        /**
+         * The map of the root node.
+         */
+        Map childMap;
+    };
+
+    /**
+     * The map type for the root node.
+     */
+    using Map = std::map<StorePath, ChildNode>;
+
+    /**
+     * The map of root node.
+     */
+    Map map;
+
+    /**
+     * Find the node for `k`, creating it if needed.
+     *
+     * The node is referred to as a "slot" on the assumption that `V` is
+     * some sort of optional type, so the given key can be set or unset
+     * by changing this node.
+     */
+    ChildNode & ensureSlot(const SingleDerivedPath & k);
+};
+
+}

src/libstore/derived-path.cc

@@ -167,7 +167,7 @@ void drvRequireExperiment(
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
     const Store & store, ref<SingleDerivedPath> drv,
-    std::string_view output,
+    OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
     drvRequireExperiment(*drv, xpSettings);
@@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 
 DerivedPath::Built DerivedPath::Built::parse(
     const Store & store, ref<SingleDerivedPath> drv,
-    std::string_view outputsS,
+    OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
     drvRequireExperiment(*drv, xpSettings);

src/libstore/derived-path.hh

@@ -42,7 +42,7 @@ struct SingleDerivedPath;
  */
 struct SingleDerivedPathBuilt {
     ref<SingleDerivedPath> drvPath;
-    std::string output;
+    OutputName output;
 
     /**
      * Get the store path this is ultimately derived from (by realising
@@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt {
      */
     static SingleDerivedPathBuilt parse(
         const Store & store, ref<SingleDerivedPath> drvPath,
-        std::string_view outputs,
+        OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
 

src/libstore/downstream-placeholder.cc

@@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const
 
 DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
     const StorePath & drvPath,
-    std::string_view outputName,
+    OutputNameView outputName,
     const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::CaDerivations);
@@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
 
 DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
     const DownstreamPlaceholder & placeholder,
-    std::string_view outputName,
+    OutputNameView outputName,
     const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::DynamicDerivations);

src/libstore/downstream-placeholder.hh

@@ -58,7 +58,7 @@ public:
      */
     static DownstreamPlaceholder unknownCaOutput(
         const StorePath & drvPath,
-        std::string_view outputName,
+        OutputNameView outputName,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
@@ -72,7 +72,7 @@ public:
      */
     static DownstreamPlaceholder unknownDerivation(
         const DownstreamPlaceholder & drvPlaceholder,
-        std::string_view outputName,
+        OutputNameView outputName,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**

src/libstore/misc.cc

@@ -399,8 +399,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
                     store.printStorePath(drvPath), bfd.output);
             auto & optPath = outputPaths.at(bfd.output);
             if (!optPath)
-                throw Error("'%s' does not yet map to a known concrete store path",
+                throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
-                    bfd.to_string(store));
             return *optPath;
         },
     }, req.raw());

src/libstore/outputs-spec.hh

@@ -13,24 +13,36 @@
 
 namespace nix {
 
+/**
+ * An (owned) output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string OutputName;
+
+/**
+ * A borrowed output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string_view OutputNameView;
+
 struct OutputsSpec {
     /**
      * A non-empty set of outputs, specified by name
      */
-    struct Names : std::set<std::string> {
+    struct Names : std::set<OutputName> {
-        using std::set<std::string>::set;
+        using std::set<OutputName>::set;
 
         /* These need to be "inherited manually" */
 
-        Names(const std::set<std::string> & s)
+        Names(const std::set<OutputName> & s)
-            : std::set<std::string>(s)
+            : std::set<OutputName>(s)
         { assert(!empty()); }
 
         /**
          * Needs to be "inherited manually"
          */
-        Names(std::set<std::string> && s)
+        Names(std::set<OutputName> && s)
-            : std::set<std::string>(s)
+            : std::set<OutputName>(s)
         { assert(!empty()); }
 
         /* This set should always be non-empty, so we delete this
@@ -57,7 +69,7 @@ struct OutputsSpec {
      */
     OutputsSpec() = delete;
 
-    bool contains(const std::string & output) const;
+    bool contains(const OutputName & output) const;
 
     /**
      * Create a new OutputsSpec which is the union of this and that.

src/libstore/realisation.hh

@@ -34,7 +34,7 @@ struct DrvOutput {
     /**
      * The name of the output.
      */
-    std::string outputName;
+    OutputName outputName;
 
     std::string to_string() const;
 
@@ -84,7 +84,7 @@ struct Realisation {
  * Since these are the outputs of a single derivation, we know the
  * output names are unique so we can use them as the map key.
  */
-typedef std::map<std::string, Realisation> SingleDrvOutputs;
+typedef std::map<OutputName, Realisation> SingleDrvOutputs;
 
 /**
  * Collection type for multiple derivations' outputs' `Realisation`s.
@@ -146,7 +146,7 @@ public:
     MissingRealisation(DrvOutput & outputId)
         : MissingRealisation(outputId.outputName, outputId.strHash())
     {}
-    MissingRealisation(std::string_view drv, std::string outputName)
+    MissingRealisation(std::string_view drv, OutputName outputName)
         : Error( "cannot operate on output '%s' of the "
                 "unbuilt derivation '%s'",
                 outputName,

tests/dyn-drv/build-built-drv.sh

@@ -18,4 +18,6 @@ clearStore
 
 drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
 
-expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
+out2=$(nix build "${drvDep}^out^out" --no-link)
+
+test $out1 == $out2