src/libstore/build/derivation-goal.cc

@@ -13,8 +13,11 @@
 
 #include <boost/outcome/try.hpp>
 #include <fstream>
+#include <kj/array.h>
 #include <kj/async-unix.h>
+#include <kj/async.h>
 #include <kj/debug.h>
+#include <kj/vector.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -173,7 +176,7 @@ try {
 
 
     state = &DerivationGoal::loadDerivation;
-    return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}};
+    return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath));
 } catch (...) {
     return {std::current_exception()};
 }
@@ -272,13 +275,13 @@ try {
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     if (settings.useSubstitutes) {
         if (parsedDrv->substitutesAllowed()) {
             for (auto & [outputName, status] : initialOutputs) {
                 if (!status.wanted) continue;
                 if (!status.known)
-                    result.goals.insert(
+                    dependencies.add(
                         worker.goalFactory().makeDrvOutputSubstitutionGoal(
                             DrvOutput{status.outputHash, outputName},
                             buildMode == bmRepair ? Repair : NoRepair
@@ -286,7 +289,7 @@ try {
                     );
                 else {
                     auto * cap = getDerivationCA(*drv);
-                    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(
+                    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
                         status.known->path,
                         buildMode == bmRepair ? Repair : NoRepair,
                         cap ? std::optional { *cap } : std::nullopt));
@@ -297,11 +300,11 @@ try {
         }
     }
 
-    if (result.goals.empty()) { /* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) { /* to prevent hang (no wake-up event) */
         return outputsSubstitutionTried(inBuildSlot);
     } else {
         state = &DerivationGoal::outputsSubstitutionTried;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -383,7 +386,7 @@ try {
    produced using a substitute.  So we have to build instead. */
 kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept
 try {
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
 
     /* At this point we are building all outputs, so if more are wanted there
        is no need to restart. */
@@ -396,7 +399,7 @@ try {
 
         addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             if (!inputNode.value.empty())
-                result.goals.insert(worker.goalFactory().makeGoal(
+                dependencies.add(worker.goalFactory().makeGoal(
                     DerivedPath::Built {
                         .drvPath = inputDrv,
                         .outputs = inputNode.value,
@@ -441,14 +444,14 @@ try {
         if (!settings.useSubstitutes)
             throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
                 worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+        dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
     }
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return inputsRealised(inBuildSlot);
     } else {
         state = &DerivationGoal::inputsRealised;
-        return {result};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -491,7 +494,7 @@ try {
         }
 
     /* Check each path (slow!). */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
         printError(
@@ -499,9 +502,9 @@ try {
             worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
         else
-            result.goals.insert(worker.goalFactory().makeGoal(
+            dependencies.add(worker.goalFactory().makeGoal(
                 DerivedPath::Built {
                     .drvPath = makeConstantStorePathRef(drvPath2->second),
                     .outputs = OutputsSpec::All { },
@@ -509,12 +512,12 @@ try {
                 bmRepair));
     }
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return {done(BuildResult::AlreadyValid, assertPathValidity())};
     }
 
     state = &DerivationGoal::closureRepaired;
-    return {result};
+    return waitForGoals(dependencies.releaseAsArray());
 } catch (...) {
     return {std::current_exception()};
 }
@@ -614,11 +617,12 @@ try {
                        worker.store.printStorePath(pathResolved),
                    });
 
-            resolvedDrvGoal = worker.goalFactory().makeDerivationGoal(
+            auto dependency = worker.goalFactory().makeDerivationGoal(
                 pathResolved, wantedOutputs, buildMode);
+            resolvedDrvGoal = dependency.first;
 
             state = &DerivationGoal::resolvedFinished;
-            return {WaitForGoals{{resolvedDrvGoal}}};
+            return waitForGoals(std::move(dependency));
         }
 
         std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;

src/libstore/build/drv-output-substitution-goal.cc

@@ -4,6 +4,9 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "signals.hh"
+#include <kj/array.h>
+#include <kj/async.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -106,7 +109,7 @@ try {
         return tryNext(inBuildSlot);
     }
 
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
         if (depId != id) {
             if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -122,17 +125,17 @@ try {
                 );
                 return tryNext(inBuildSlot);
             }
-            result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
+            dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
         }
     }
 
-    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
+    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return outPathValid(inBuildSlot);
     } else {
         state = &DrvOutputSubstitutionGoal::outPathValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};

src/libstore/build/entry-points.cc

@@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     Worker worker(*this, evalStore ? *evalStore : *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (auto & br : reqs)
-            goals.insert(gf.makeGoal(br, buildMode));
+            goals.emplace(gf.makeGoal(br, buildMode));
         return goals;
     });
 
@@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
     std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (const auto & req : reqs) {
             auto goal = gf.makeGoal(req, buildMode);
-            goals.insert(goal);
+            state.push_back({req, goal.first});
-            state.push_back({req, goal});
+            goals.emplace(std::move(goal));
         }
         return goals;
     });
@@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     Worker worker(*this, *this, aio);
 
     try {
-        auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals {
+        auto goals = runWorker(worker, [&](GoalFactory & gf) {
-            return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)};
+            Worker::Targets goals;
+            goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode));
+            return goals;
         });
         auto goal = *goals.begin();
         return goal->buildResult.restrictTo(DerivedPath::Built {
@@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path, Repair)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path, Repair));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             worker.run([&](GoalFactory & gf) {
-                return Goals{gf.makeGoal(
+                Worker::Targets goals;
+                goals.emplace(gf.makeGoal(
                     DerivedPath::Built{
                         .drvPath = makeConstantStorePathRef(*info->deriver),
                         // FIXME: Should just build the specific output we need.
                         .outputs = OutputsSpec::All{},
                     },
                     bmRepair
-                )};
+                ));
+                return goals;
             });
         } else
             throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));

src/libstore/build/goal.cc

@@ -1,4 +1,5 @@
 #include "goal.hh"
+#include "async-collect.hh"
 #include "worker.hh"
 #include <kj/time.h>
 
@@ -28,4 +29,32 @@ try {
     co_return std::current_exception();
 }
 
+kj::Promise<Result<Goal::WorkResult>>
+Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept
+try {
+    auto left = dependencies.size();
+    auto collectDeps = asyncCollect(std::move(dependencies));
+
+    while (auto item = co_await collectDeps.next()) {
+        left--;
+        auto & dep = *item;
+
+        trace(fmt("waitee '%s' done; %d left", dep->name, left));
+
+        if (dep->exitCode != Goal::ecSuccess) ++nrFailed;
+        if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters;
+        if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure;
+
+        waiteeDone(dep);
+
+        if (dep->exitCode == ecFailed && !settings.keepGoing) {
+            co_return result::success(ContinueImmediately{});
+        }
+    }
+
+    co_return result::success(ContinueImmediately{});
+} catch (...) {
+    co_return result::failure(std::current_exception());
+}
+
 }

src/libstore/build/goal.hh

@@ -6,6 +6,7 @@
 #include "types.hh"
 #include "store-api.hh"
 #include "build-result.hh"
+#include <concepts> // IWYU pragma: keep
 #include <kj/async.h>
 
 namespace nix {
@@ -70,17 +71,6 @@ struct Goal
       */
     const bool isDependency;
 
-    /**
-     * Goals that this goal is waiting for.
-     */
-    Goals waitees;
-
-    /**
-     * Goals waiting for this one to finish.  Must use weak pointers
-     * here to prevent cycles.
-     */
-    WeakGoals waiters;
-
     /**
      * Number of goals we are/were waiting for that have failed.
      */
@@ -113,6 +103,9 @@ struct Goal
      */
     BuildResult buildResult;
 
+    // for use by Worker only. will go away once work() is a promise.
+    kj::Own<kj::PromiseFulfiller<void>> notify;
+
 protected:
     AsyncSemaphore::Token slotToken;
 
@@ -122,9 +115,6 @@ public:
 
     struct [[nodiscard]] StillAlive {};
     struct [[nodiscard]] ContinueImmediately {};
-    struct [[nodiscard]] WaitForGoals {
-        Goals goals;
-    };
     struct [[nodiscard]] WaitForWorld {
         kj::Promise<Outcome<void, Finished>> promise;
     };
@@ -141,7 +131,6 @@ public:
     struct [[nodiscard]] WorkResult : std::variant<
                                           StillAlive,
                                           ContinueImmediately,
-                                          WaitForGoals,
                                           WaitForWorld,
                                           Finished>
     {
@@ -151,6 +140,15 @@ public:
 
 protected:
     kj::Promise<Result<WorkResult>> waitForAWhile();
+    kj::Promise<Result<WorkResult>>
+    waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept;
+
+    template<std::derived_from<Goal>... G>
+    kj::Promise<Result<Goal::WorkResult>>
+    waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<void>>... goals) noexcept
+    {
+        return waitForGoals(kj::arrOf<std::pair<GoalPtr, kj::Promise<void>>>(std::move(goals)...));
+    }
 
 public:
 

src/libstore/build/substitution-goal.cc

@@ -3,6 +3,8 @@
 #include "nar-info.hh"
 #include "signals.hh"
 #include "finally.hh"
+#include <kj/array.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -160,16 +162,16 @@ try {
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return referencesValid(inBuildSlot);
     } else {
         state = &PathSubstitutionGoal::referencesValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};

src/libstore/build/worker.cc

@@ -52,26 +52,28 @@ Worker::~Worker()
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
     std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
-    std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
+    auto & goal_weak = derivationGoals[drvPath];
-    std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+    std::shared_ptr<DerivationGoal> goal = goal_weak.goal.lock();
     if (!goal) {
         goal = mkDrvGoal();
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     } else {
         goal->addWantedOutputs(wantedOutputs);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoal(
-    const OutputsSpec & wantedOutputs, BuildMode buildMode)
+    const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -89,8 +91,12 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeBasicDerivationGoal(
-    const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
+    const StorePath & drvPath,
+    const BasicDerivation & drv,
+    const OutputsSpec & wantedOutputs,
+    BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -108,55 +114,63 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 }
 
 
-std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+Worker::makePathSubstitutionGoal(
+    const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
+    auto & goal_weak = substitutionGoals[path];
-    auto goal = goal_weak.lock(); // FIXME
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+Worker::makeDrvOutputSubstitutionGoal(
+    const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
+    auto & goal_weak = drvOutputSubstitutionGoals[id];
-    auto goal = goal_weak.lock(); // FIXME
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
+std::pair<GoalPtr, kj::Promise<void>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & bfd) -> GoalPtr {
+        [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<void>> {
             if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
                 return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
             else
                 throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
-        [&](const DerivedPath::Opaque & bo) -> GoalPtr {
+        [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<void>> {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
         },
     }, req.raw());
 }
 
 
-template<typename K, typename G>
+template<typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
+static void removeGoal(std::shared_ptr<G> goal, auto & goalMap)
 {
     /* !!! inefficient */
     for (auto i = goalMap.begin();
          i != goalMap.end(); )
-        if (i->second.lock() == goal) {
+        if (i->second.goal.lock() == goal) {
             auto j = i; ++j;
             goalMap.erase(i);
             i = j;
@@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f)
     hashMismatch |= f.hashMismatch;
     checkMismatch |= f.checkMismatch;
 
-    for (auto & i : goal->waiters) {
-        if (GoalPtr waiting = i.lock()) {
-            assert(waiting->waitees.count(goal));
-            waiting->waitees.erase(goal);
-
-            waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size()));
-
-            if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed;
-            if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters;
-            if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure;
-
-            if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) {
-                /* If we failed and keepGoing is not set, we remove all
-                   remaining waitees. */
-                for (auto & i : waiting->waitees) {
-                    i->waiters.extract(waiting);
-                }
-                waiting->waitees.clear();
-
-                wakeUp(waiting);
-            }
-
-            waiting->waiteeDone(goal);
-        }
-    }
-    goal->waiters.clear();
     removeGoal(goal);
+    goal->notify->fulfill();
     goal->cleanup();
 }
 
@@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how)
         overloaded{
             [&](Goal::StillAlive) {},
             [&](Goal::ContinueImmediately) { wakeUp(goal); },
-            [&](Goal::WaitForGoals & w) {
-                for (auto & dep : w.goals) {
-                    goal->waitees.insert(dep);
-                    dep->waiters.insert(goal);
-                }
-            },
             [&](Goal::WaitForWorld & w) {
                 childStarted(goal, w.promise.then([](auto r) -> Result<Goal::WorkResult> {
                     if (r.has_value()) {
@@ -310,7 +293,7 @@ void Worker::updateStatistics()
     }
 }
 
-Goals Worker::run(std::function<Goals (GoalFactory &)> req)
+std::vector<GoalPtr> Worker::run(std::function<Targets (GoalFactory &)> req)
 {
     auto _topGoals = req(goalFactory());
 
@@ -320,7 +303,10 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
 
     updateStatistics();
 
-    topGoals = _topGoals;
+    topGoals.clear();
+    for (auto & [goal, _promise] : _topGoals) {
+        topGoals.insert(goal);
+    }
 
     debug("entered goal loop");
 
@@ -374,7 +360,11 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
     assert(!settings.keepGoing || awake.empty());
     assert(!settings.keepGoing || children.isEmpty());
 
-    return _topGoals;
+    std::vector<GoalPtr> results;
+    for (auto & [i, _p] : _topGoals) {
+        results.push_back(i);
+    }
+    return results;
 }
 
 void Worker::waitForInput()

src/libstore/build/worker.hh

@@ -28,10 +28,10 @@ struct HookInstance;
 class GoalFactory
 {
 public:
-    virtual std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal
     ) = 0;
-    virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath,
         const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs,
@@ -41,12 +41,14 @@ public:
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+    makePathSubstitutionGoal(
         const StorePath & storePath,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
     ) = 0;
-    virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
         const DrvOutput & id,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
@@ -58,7 +60,8 @@ public:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
+    virtual std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
 };
 
 // elaborate hoax to let goals access factory methods while hiding them from the public
@@ -94,13 +97,27 @@ private:
      */
     WeakGoals awake;
 
+    template<typename G>
+    struct CachedGoal
+    {
+        std::weak_ptr<G> goal;
+        kj::Own<kj::ForkedPromise<void>> promise;
+        kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+
+        CachedGoal()
+        {
+            auto pf = kj::newPromiseAndFulfiller<void>();
+            promise = kj::heap(pf.promise.fork());
+            fulfiller = std::move(pf.fulfiller);
+        }
+    };
     /**
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
-    std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
+    std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals;
-    std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
+    std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals;
-    std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
+    std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
 
     /**
      * Cache for pathContentsGood().
@@ -226,21 +243,31 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath, const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
 
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+    std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
-    std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+    makePathSubstitutionGoal(
+        const StorePath & storePath,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
+    std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
+        const DrvOutput & id,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
 
     /**
      * Make a goal corresponding to the `DerivedPath`.
@@ -248,13 +275,16 @@ private:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
+    std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
 
 public:
+    using Targets = std::map<GoalPtr, kj::Promise<void>>;
+
     /**
      * Loop until the specified top-level goals have finished.
      */
-    Goals run(std::function<Goals (GoalFactory &)> req);
+    std::vector<GoalPtr> run(std::function<Targets (GoalFactory &)> req);
 
     /***
      * The exit status in case of failure.