libstore: turn Goal::WaitForGoals into a promise

also gets rid of explicit strong references to dependencies of any goal,
and weak references to dependers as well. those are now only held within
promises representing goal completion and thus independent of the goal's
relation to each other. the weak references to dependers was only needed
for notifications, and that's much better handled entirely by kj itself.

Change-Id: I00d06df9090f8d6336ee4bb0c1313a7052fb016b

src/libstore/build/derivation-goal.cc

@@ -13,8 +13,11 @@
 
 #include <boost/outcome/try.hpp>
 #include <fstream>
+#include <kj/array.h>
 #include <kj/async-unix.h>
+#include <kj/async.h>
 #include <kj/debug.h>
+#include <kj/vector.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -173,7 +176,7 @@ try {
 
 
     state = &DerivationGoal::loadDerivation;
-    return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}};
+    return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath));
 } catch (...) {
     return {std::current_exception()};
 }
@@ -272,13 +275,13 @@ try {
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     if (settings.useSubstitutes) {
         if (parsedDrv->substitutesAllowed()) {
             for (auto & [outputName, status] : initialOutputs) {
                 if (!status.wanted) continue;
                 if (!status.known)
-                    result.goals.insert(
+                    dependencies.add(
                         worker.goalFactory().makeDrvOutputSubstitutionGoal(
                             DrvOutput{status.outputHash, outputName},
                             buildMode == bmRepair ? Repair : NoRepair
@@ -286,7 +289,7 @@ try {
                     );
                 else {
                     auto * cap = getDerivationCA(*drv);
-                    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(
+                    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
                         status.known->path,
                         buildMode == bmRepair ? Repair : NoRepair,
                         cap ? std::optional { *cap } : std::nullopt));
@@ -297,11 +300,11 @@ try {
         }
     }
 
-    if (result.goals.empty()) { /* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) { /* to prevent hang (no wake-up event) */
         return outputsSubstitutionTried(inBuildSlot);
     } else {
         state = &DerivationGoal::outputsSubstitutionTried;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -383,7 +386,7 @@ try {
    produced using a substitute.  So we have to build instead. */
 kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept
 try {
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
 
     /* At this point we are building all outputs, so if more are wanted there
        is no need to restart. */
@@ -396,7 +399,7 @@ try {
 
         addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             if (!inputNode.value.empty())
-                result.goals.insert(worker.goalFactory().makeGoal(
+                dependencies.add(worker.goalFactory().makeGoal(
                     DerivedPath::Built {
                         .drvPath = inputDrv,
                         .outputs = inputNode.value,
@@ -441,14 +444,14 @@ try {
         if (!settings.useSubstitutes)
             throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
                 worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+        dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
     }
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return inputsRealised(inBuildSlot);
     } else {
         state = &DerivationGoal::inputsRealised;
-        return {result};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -491,7 +494,7 @@ try {
         }
 
     /* Check each path (slow!). */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
         printError(
@@ -499,9 +502,9 @@ try {
             worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
         else
-            result.goals.insert(worker.goalFactory().makeGoal(
+            dependencies.add(worker.goalFactory().makeGoal(
                 DerivedPath::Built {
                     .drvPath = makeConstantStorePathRef(drvPath2->second),
                     .outputs = OutputsSpec::All { },
@@ -509,12 +512,12 @@ try {
                 bmRepair));
     }
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return {done(BuildResult::AlreadyValid, assertPathValidity())};
     }
 
     state = &DerivationGoal::closureRepaired;
-    return {result};
+    return waitForGoals(dependencies.releaseAsArray());
 } catch (...) {
     return {std::current_exception()};
 }
@@ -614,11 +617,12 @@ try {
                        worker.store.printStorePath(pathResolved),
                    });
 
-            resolvedDrvGoal = worker.goalFactory().makeDerivationGoal(
+            auto dependency = worker.goalFactory().makeDerivationGoal(
                 pathResolved, wantedOutputs, buildMode);
+            resolvedDrvGoal = dependency.first;
 
             state = &DerivationGoal::resolvedFinished;
-            return {WaitForGoals{{resolvedDrvGoal}}};
+            return waitForGoals(std::move(dependency));
         }
 
         std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;

src/libstore/build/drv-output-substitution-goal.cc

@@ -4,6 +4,9 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "signals.hh"
+#include <kj/array.h>
+#include <kj/async.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -106,7 +109,7 @@ try {
         return tryNext(inBuildSlot);
     }
 
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
         if (depId != id) {
             if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -122,17 +125,17 @@ try {
                 );
                 return tryNext(inBuildSlot);
             }
-            result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
+            dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
         }
     }
 
-    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
+    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return outPathValid(inBuildSlot);
     } else {
         state = &DrvOutputSubstitutionGoal::outPathValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};

src/libstore/build/entry-points.cc

@@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     Worker worker(*this, evalStore ? *evalStore : *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (auto & br : reqs)
-            goals.insert(gf.makeGoal(br, buildMode));
+            goals.emplace(gf.makeGoal(br, buildMode));
         return goals;
     });
 
@@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
     std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (const auto & req : reqs) {
             auto goal = gf.makeGoal(req, buildMode);
-            goals.insert(goal);
+            state.push_back({req, goal.first});
-            state.push_back({req, goal});
+            goals.emplace(std::move(goal));
         }
         return goals;
     });
@@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     Worker worker(*this, *this, aio);
 
     try {
-        auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals {
+        auto goals = runWorker(worker, [&](GoalFactory & gf) {
-            return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)};
+            Worker::Targets goals;
+            goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode));
+            return goals;
         });
         auto goal = *goals.begin();
         return goal->buildResult.restrictTo(DerivedPath::Built {
@@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path, Repair)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path, Repair));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             worker.run([&](GoalFactory & gf) {
-                return Goals{gf.makeGoal(
+                Worker::Targets goals;
+                goals.emplace(gf.makeGoal(
                     DerivedPath::Built{
                         .drvPath = makeConstantStorePathRef(*info->deriver),
                         // FIXME: Should just build the specific output we need.
                         .outputs = OutputsSpec::All{},
                     },
                     bmRepair
-                )};
+                ));
+                return goals;
             });
         } else
             throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));

src/libstore/build/goal.cc

@@ -1,4 +1,5 @@
 #include "goal.hh"
+#include "async-collect.hh"
 #include "worker.hh"
 #include <kj/time.h>
 
@@ -28,4 +29,32 @@ try {
     co_return std::current_exception();
 }
 
+kj::Promise<Result<Goal::WorkResult>>
+Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept
+try {
+    auto left = dependencies.size();
+    auto collectDeps = asyncCollect(std::move(dependencies));
+
+    while (auto item = co_await collectDeps.next()) {
+        left--;
+        auto & dep = *item;
+
+        trace(fmt("waitee '%s' done; %d left", dep->name, left));
+
+        if (dep->exitCode != Goal::ecSuccess) ++nrFailed;
+        if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters;
+        if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure;
+
+        waiteeDone(dep);
+
+        if (dep->exitCode == ecFailed && !settings.keepGoing) {
+            co_return result::success(ContinueImmediately{});
+        }
+    }
+
+    co_return result::success(ContinueImmediately{});
+} catch (...) {
+    co_return result::failure(std::current_exception());
+}
+
 }

src/libstore/build/goal.hh

@@ -6,6 +6,7 @@
 #include "types.hh"
 #include "store-api.hh"
 #include "build-result.hh"
+#include <concepts> // IWYU pragma: keep
 #include <kj/async.h>
 
 namespace nix {
@@ -70,17 +71,6 @@ struct Goal
       */
     const bool isDependency;
 
-    /**
-     * Goals that this goal is waiting for.
-     */
-    Goals waitees;
-
-    /**
-     * Goals waiting for this one to finish.  Must use weak pointers
-     * here to prevent cycles.
-     */
-    WeakGoals waiters;
-
     /**
      * Number of goals we are/were waiting for that have failed.
      */
@@ -113,6 +103,9 @@ struct Goal
      */
     BuildResult buildResult;
 
+    // for use by Worker only. will go away once work() is a promise.
+    kj::Own<kj::PromiseFulfiller<void>> notify;
+
 protected:
     AsyncSemaphore::Token slotToken;
 
@@ -122,9 +115,6 @@ public:
 
     struct [[nodiscard]] StillAlive {};
     struct [[nodiscard]] ContinueImmediately {};
-    struct [[nodiscard]] WaitForGoals {
-        Goals goals;
-    };
     struct [[nodiscard]] WaitForWorld {
         kj::Promise<Outcome<void, Finished>> promise;
     };
@@ -141,7 +131,6 @@ public:
     struct [[nodiscard]] WorkResult : std::variant<
                                           StillAlive,
                                           ContinueImmediately,
-                                          WaitForGoals,
                                           WaitForWorld,
                                           Finished>
     {
@@ -151,6 +140,15 @@ public:
 
 protected:
     kj::Promise<Result<WorkResult>> waitForAWhile();
+    kj::Promise<Result<WorkResult>>
+    waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept;
+
+    template<std::derived_from<Goal>... G>
+    kj::Promise<Result<Goal::WorkResult>>
+    waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<void>>... goals) noexcept
+    {
+        return waitForGoals(kj::arrOf<std::pair<GoalPtr, kj::Promise<void>>>(std::move(goals)...));
+    }
 
 public:
 

src/libstore/build/substitution-goal.cc

@@ -3,6 +3,8 @@
 #include "nar-info.hh"
 #include "signals.hh"
 #include "finally.hh"
+#include <kj/array.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -160,16 +162,16 @@ try {
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return referencesValid(inBuildSlot);
     } else {
         state = &PathSubstitutionGoal::referencesValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};

src/libstore/build/worker.cc

@@ -52,26 +52,28 @@ Worker::~Worker()
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
     std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
-    std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
+    auto & goal_weak = derivationGoals[drvPath];
-    std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+    std::shared_ptr<DerivationGoal> goal = goal_weak.goal.lock();
     if (!goal) {
         goal = mkDrvGoal();
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     } else {
         goal->addWantedOutputs(wantedOutputs);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoal(
-    const OutputsSpec & wantedOutputs, BuildMode buildMode)
+    const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -89,8 +91,12 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeBasicDerivationGoal(
-    const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
+    const StorePath & drvPath,
+    const BasicDerivation & drv,
+    const OutputsSpec & wantedOutputs,
+    BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -108,55 +114,63 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 }
 
 
-std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+Worker::makePathSubstitutionGoal(
+    const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
+    auto & goal_weak = substitutionGoals[path];
-    auto goal = goal_weak.lock(); // FIXME
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+Worker::makeDrvOutputSubstitutionGoal(
+    const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
+    auto & goal_weak = drvOutputSubstitutionGoals[id];
-    auto goal = goal_weak.lock(); // FIXME
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
+std::pair<GoalPtr, kj::Promise<void>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & bfd) -> GoalPtr {
+        [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<void>> {
             if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
                 return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
             else
                 throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
-        [&](const DerivedPath::Opaque & bo) -> GoalPtr {
+        [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<void>> {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
         },
     }, req.raw());
 }
 
 
-template<typename K, typename G>
+template<typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
+static void removeGoal(std::shared_ptr<G> goal, auto & goalMap)
 {
     /* !!! inefficient */
     for (auto i = goalMap.begin();
          i != goalMap.end(); )
-        if (i->second.lock() == goal) {
+        if (i->second.goal.lock() == goal) {
             auto j = i; ++j;
             goalMap.erase(i);
             i = j;
@@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f)
     hashMismatch |= f.hashMismatch;
     checkMismatch |= f.checkMismatch;
 
-    for (auto & i : goal->waiters) {
-        if (GoalPtr waiting = i.lock()) {
-            assert(waiting->waitees.count(goal));
-            waiting->waitees.erase(goal);
-
-            waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size()));
-
-            if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed;
-            if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters;
-            if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure;
-
-            if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) {
-                /* If we failed and keepGoing is not set, we remove all
-                   remaining waitees. */
-                for (auto & i : waiting->waitees) {
-                    i->waiters.extract(waiting);
-                }
-                waiting->waitees.clear();
-
-                wakeUp(waiting);
-            }
-
-            waiting->waiteeDone(goal);
-        }
-    }
-    goal->waiters.clear();
     removeGoal(goal);
+    goal->notify->fulfill();
     goal->cleanup();
 }
 
@@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how)
         overloaded{
             [&](Goal::StillAlive) {},
             [&](Goal::ContinueImmediately) { wakeUp(goal); },
-            [&](Goal::WaitForGoals & w) {
-                for (auto & dep : w.goals) {
-                    goal->waitees.insert(dep);
-                    dep->waiters.insert(goal);
-                }
-            },
             [&](Goal::WaitForWorld & w) {
                 childStarted(goal, w.promise.then([](auto r) -> Result<Goal::WorkResult> {
                     if (r.has_value()) {
@@ -310,7 +293,7 @@ void Worker::updateStatistics()
     }
 }
 
-Goals Worker::run(std::function<Goals (GoalFactory &)> req)
+std::vector<GoalPtr> Worker::run(std::function<Targets (GoalFactory &)> req)
 {
     auto _topGoals = req(goalFactory());
 
@@ -320,7 +303,10 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
 
     updateStatistics();
 
-    topGoals = _topGoals;
+    topGoals.clear();
+    for (auto & [goal, _promise] : _topGoals) {
+        topGoals.insert(goal);
+    }
 
     debug("entered goal loop");
 
@@ -374,7 +360,11 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
     assert(!settings.keepGoing || awake.empty());
     assert(!settings.keepGoing || children.isEmpty());
 
-    return _topGoals;
+    std::vector<GoalPtr> results;
+    for (auto & [i, _p] : _topGoals) {
+        results.push_back(i);
+    }
+    return results;
 }
 
 void Worker::waitForInput()

src/libstore/build/worker.hh

@@ -28,10 +28,10 @@ struct HookInstance;
 class GoalFactory
 {
 public:
-    virtual std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal
     ) = 0;
-    virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath,
         const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs,
@@ -41,12 +41,14 @@ public:
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+    makePathSubstitutionGoal(
         const StorePath & storePath,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
     ) = 0;
-    virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
         const DrvOutput & id,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
@@ -58,7 +60,8 @@ public:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
+    virtual std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
 };
 
 // elaborate hoax to let goals access factory methods while hiding them from the public
@@ -94,13 +97,27 @@ private:
      */
     WeakGoals awake;
 
+    template<typename G>
+    struct CachedGoal
+    {
+        std::weak_ptr<G> goal;
+        kj::Own<kj::ForkedPromise<void>> promise;
+        kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+
+        CachedGoal()
+        {
+            auto pf = kj::newPromiseAndFulfiller<void>();
+            promise = kj::heap(pf.promise.fork());
+            fulfiller = std::move(pf.fulfiller);
+        }
+    };
     /**
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
-    std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
+    std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals;
-    std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
+    std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals;
-    std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
+    std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
 
     /**
      * Cache for pathContentsGood().
@@ -226,21 +243,31 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath, const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
 
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+    std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
-    std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+    makePathSubstitutionGoal(
+        const StorePath & storePath,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
+    std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
+        const DrvOutput & id,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
 
     /**
      * Make a goal corresponding to the `DerivedPath`.
@@ -248,13 +275,16 @@ private:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
+    std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
 
 public:
+    using Targets = std::map<GoalPtr, kj::Promise<void>>;
+
     /**
      * Loop until the specified top-level goals have finished.
      */
-    Goals run(std::function<Goals (GoalFactory &)> req);
+    std::vector<GoalPtr> run(std::function<Targets (GoalFactory &)> req);
 
     /***
      * The exit status in case of failure.