From ae5d8dae1b796967c40b1f22b844d07f5697033e Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH] libstore: turn Goal::WaitForGoals into a promise also gets rid of explicit strong references to dependencies of any goal, and weak references to dependers as well. those are now only held within promises representing goal completion and thus independent of the goal's relation to each other. the weak references to dependers was only needed for notifications, and that's much better handled entirely by kj itself. Change-Id: I00d06df9090f8d6336ee4bb0c1313a7052fb016b --- src/libstore/build/derivation-goal.cc | 40 ++++--- .../build/drv-output-substitution-goal.cc | 13 ++- src/libstore/build/entry-points.cc | 30 +++-- src/libstore/build/goal.cc | 29 +++++ src/libstore/build/goal.hh | 28 +++-- src/libstore/build/substitution-goal.cc | 10 +- src/libstore/build/worker.cc | 110 ++++++++---------- src/libstore/build/worker.hh | 60 +++++++--- 8 files changed, 192 insertions(+), 128 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3c4257f08..b8c4d278d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -13,8 +13,11 @@ #include #include +#include #include +#include #include +#include #include #include #include @@ -173,7 +176,7 @@ try { state = &DerivationGoal::loadDerivation; - return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}}; + return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath)); } catch (...) { return {std::current_exception()}; } @@ -272,13 +275,13 @@ try { /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - WaitForGoals result; + kj::Vector>> dependencies; if (settings.useSubstitutes) { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - result.goals.insert( + dependencies.add( worker.goalFactory().makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair @@ -286,7 +289,7 @@ try { ); else { auto * cap = getDerivationCA(*drv); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal( + dependencies.add(worker.goalFactory().makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); @@ -297,11 +300,11 @@ try { } } - if (result.goals.empty()) { /* to prevent hang (no wake-up event) */ + if (dependencies.empty()) { /* to prevent hang (no wake-up event) */ return outputsSubstitutionTried(inBuildSlot); } else { state = &DerivationGoal::outputsSubstitutionTried; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; @@ -383,7 +386,7 @@ try { produced using a substitute. So we have to build instead. */ kj::Promise> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept try { - WaitForGoals result; + kj::Vector>> dependencies; /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -396,7 +399,7 @@ try { addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -441,14 +444,14 @@ try { if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); } - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ + if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ return inputsRealised(inBuildSlot); } else { state = &DerivationGoal::inputsRealised; - return {result}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; @@ -491,7 +494,7 @@ try { } /* Check each path (slow!). */ - WaitForGoals result; + kj::Vector>> dependencies; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( @@ -499,9 +502,9 @@ try { worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); else - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -509,12 +512,12 @@ try { bmRepair)); } - if (result.goals.empty()) { + if (dependencies.empty()) { return {done(BuildResult::AlreadyValid, assertPathValidity())}; } state = &DerivationGoal::closureRepaired; - return {result}; + return waitForGoals(dependencies.releaseAsArray()); } catch (...) { return {std::current_exception()}; } @@ -614,11 +617,12 @@ try { worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.goalFactory().makeDerivationGoal( + auto dependency = worker.goalFactory().makeDerivationGoal( pathResolved, wantedOutputs, buildMode); + resolvedDrvGoal = dependency.first; state = &DerivationGoal::resolvedFinished; - return {WaitForGoals{{resolvedDrvGoal}}}; + return waitForGoals(std::move(dependency)); } std::function::ChildNode &)> accumInputPaths; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 6ef00d1ff..80b2c4cfb 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -4,6 +4,9 @@ #include "worker.hh" #include "substitution-goal.hh" #include "signals.hh" +#include +#include +#include namespace nix { @@ -106,7 +109,7 @@ try { return tryNext(inBuildSlot); } - WaitForGoals result; + kj::Vector>> dependencies; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -122,17 +125,17 @@ try { ); return tryNext(inBuildSlot); } - result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); + dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } } - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - if (result.goals.empty()) { + if (dependencies.empty()) { return outPathValid(inBuildSlot); } else { state = &DrvOutputSubstitutionGoal::outPathValid; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a0f18a02c..27c341295 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (auto & br : reqs) - goals.insert(gf.makeGoal(br, buildMode)); + goals.emplace(gf.makeGoal(br, buildMode)); return goals; }); @@ -60,11 +60,11 @@ std::vector Store::buildPathsWithResults( std::vector> state; auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (const auto & req : reqs) { auto goal = gf.makeGoal(req, buildMode); - goals.insert(goal); - state.push_back({req, goal}); + state.push_back({req, goal.first}); + goals.emplace(std::move(goal)); } return goals; }); @@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat Worker worker(*this, *this, aio); try { - auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals { - return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; + auto goals = runWorker(worker, [&](GoalFactory & gf) { + Worker::Targets goals; + goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + return goals; }); auto goal = *goals.begin(); return goal->buildResult.restrictTo(DerivedPath::Built { @@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path)); + return goals; }); auto goal = *goals.begin(); @@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path, Repair)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); + return goals; }); auto goal = *goals.begin(); @@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { worker.run([&](GoalFactory & gf) { - return Goals{gf.makeGoal( + Worker::Targets goals; + goals.emplace(gf.makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair - )}; + )); + return goals; }); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 649093dbd..8a2f4ab35 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,4 +1,5 @@ #include "goal.hh" +#include "async-collect.hh" #include "worker.hh" #include @@ -28,4 +29,32 @@ try { co_return std::current_exception(); } +kj::Promise> +Goal::waitForGoals(kj::Array>> dependencies) noexcept +try { + auto left = dependencies.size(); + auto collectDeps = asyncCollect(std::move(dependencies)); + + while (auto item = co_await collectDeps.next()) { + left--; + auto & dep = *item; + + trace(fmt("waitee '%s' done; %d left", dep->name, left)); + + if (dep->exitCode != Goal::ecSuccess) ++nrFailed; + if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + + waiteeDone(dep); + + if (dep->exitCode == ecFailed && !settings.keepGoing) { + co_return result::success(ContinueImmediately{}); + } + } + + co_return result::success(ContinueImmediately{}); +} catch (...) { + co_return result::failure(std::current_exception()); +} + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1ccf9716b..e7a500a00 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include // IWYU pragma: keep #include namespace nix { @@ -70,17 +71,6 @@ struct Goal */ const bool isDependency; - /** - * Goals that this goal is waiting for. - */ - Goals waitees; - - /** - * Goals waiting for this one to finish. Must use weak pointers - * here to prevent cycles. - */ - WeakGoals waiters; - /** * Number of goals we are/were waiting for that have failed. */ @@ -113,6 +103,9 @@ struct Goal */ BuildResult buildResult; + // for use by Worker only. will go away once work() is a promise. + kj::Own> notify; + protected: AsyncSemaphore::Token slotToken; @@ -122,9 +115,6 @@ public: struct [[nodiscard]] StillAlive {}; struct [[nodiscard]] ContinueImmediately {}; - struct [[nodiscard]] WaitForGoals { - Goals goals; - }; struct [[nodiscard]] WaitForWorld { kj::Promise> promise; }; @@ -141,7 +131,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, ContinueImmediately, - WaitForGoals, WaitForWorld, Finished> { @@ -151,6 +140,15 @@ public: protected: kj::Promise> waitForAWhile(); + kj::Promise> + waitForGoals(kj::Array>> dependencies) noexcept; + + template... G> + kj::Promise> + waitForGoals(std::pair, kj::Promise>... goals) noexcept + { + return waitForGoals(kj::arrOf>>(std::move(goals)...)); + } public: diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 6d90196fa..74a63ca21 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,8 @@ #include "nar-info.hh" #include "signals.hh" #include "finally.hh" +#include +#include namespace nix { @@ -160,16 +162,16 @@ try { /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - WaitForGoals result; + kj::Vector>> dependencies; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ + if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ return referencesValid(inBuildSlot); } else { state = &PathSubstitutionGoal::referencesValid; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e19917d91..68071a94c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -52,26 +52,28 @@ Worker::~Worker() } -std::shared_ptr Worker::makeDerivationGoalCommon( +std::pair, kj::Promise> Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal) { - std::weak_ptr & goal_weak = derivationGoals[drvPath]; - std::shared_ptr goal = goal_weak.lock(); + auto & goal_weak = derivationGoals[drvPath]; + std::shared_ptr goal = goal_weak.goal.lock(); if (!goal) { goal = mkDrvGoal(); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } else { goal->addWantedOutputs(wantedOutputs); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair, kj::Promise> Worker::makeDerivationGoal( + const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode +) { return makeDerivationGoalCommon( drvPath, @@ -89,8 +91,12 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv } -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair, kj::Promise> Worker::makeBasicDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + BuildMode buildMode +) { return makeDerivationGoalCommon( drvPath, @@ -108,55 +114,63 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::pair, kj::Promise> +Worker::makePathSubstitutionGoal( + const StorePath & path, RepairFlag repair, std::optional ca +) { - std::weak_ptr & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME + auto & goal_weak = substitutionGoals[path]; + auto goal = goal_weak.goal.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, running, repair, ca); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +std::pair, kj::Promise> +Worker::makeDrvOutputSubstitutionGoal( + const DrvOutput & id, RepairFlag repair, std::optional ca +) { - std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; - auto goal = goal_weak.lock(); // FIXME + auto & goal_weak = drvOutputSubstitutionGoals[id]; + auto goal = goal_weak.goal.lock(); // FIXME if (!goal) { goal = std::make_shared(id, *this, running, repair, ca); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +std::pair> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> GoalPtr { + [&](const DerivedPath::Built & bfd) -> std::pair> { if (auto bop = std::get_if(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, - [&](const DerivedPath::Opaque & bo) -> GoalPtr { + [&](const DerivedPath::Opaque & bo) -> std::pair> { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); } -template -static void removeGoal(std::shared_ptr goal, std::map> & goalMap) +template +static void removeGoal(std::shared_ptr goal, auto & goalMap) { /* !!! inefficient */ for (auto i = goalMap.begin(); i != goalMap.end(); ) - if (i->second.lock() == goal) { + if (i->second.goal.lock() == goal) { auto j = i; ++j; goalMap.erase(i); i = j; @@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; - for (auto & i : goal->waiters) { - if (GoalPtr waiting = i.lock()) { - assert(waiting->waitees.count(goal)); - waiting->waitees.erase(goal); - - waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size())); - - if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed; - if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; - if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; - - if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) { - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & i : waiting->waitees) { - i->waiters.extract(waiting); - } - waiting->waitees.clear(); - - wakeUp(waiting); - } - - waiting->waiteeDone(goal); - } - } - goal->waiters.clear(); removeGoal(goal); + goal->notify->fulfill(); goal->cleanup(); } @@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) overloaded{ [&](Goal::StillAlive) {}, [&](Goal::ContinueImmediately) { wakeUp(goal); }, - [&](Goal::WaitForGoals & w) { - for (auto & dep : w.goals) { - goal->waitees.insert(dep); - dep->waiters.insert(goal); - } - }, [&](Goal::WaitForWorld & w) { childStarted(goal, w.promise.then([](auto r) -> Result { if (r.has_value()) { @@ -310,7 +293,7 @@ void Worker::updateStatistics() } } -Goals Worker::run(std::function req) +std::vector Worker::run(std::function req) { auto _topGoals = req(goalFactory()); @@ -320,7 +303,10 @@ Goals Worker::run(std::function req) updateStatistics(); - topGoals = _topGoals; + topGoals.clear(); + for (auto & [goal, _promise] : _topGoals) { + topGoals.insert(goal); + } debug("entered goal loop"); @@ -374,7 +360,11 @@ Goals Worker::run(std::function req) assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || children.isEmpty()); - return _topGoals; + std::vector results; + for (auto & [i, _p] : _topGoals) { + results.push_back(i); + } + return results; } void Worker::waitForInput() diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 834ecfda3..925d289bf 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -28,10 +28,10 @@ struct HookInstance; class GoalFactory { public: - virtual std::shared_ptr makeDerivationGoal( + virtual std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal ) = 0; - virtual std::shared_ptr makeBasicDerivationGoal( + virtual std::pair, kj::Promise> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -41,12 +41,14 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - virtual std::shared_ptr makePathSubstitutionGoal( + virtual std::pair, kj::Promise> + makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) = 0; - virtual std::shared_ptr makeDrvOutputSubstitutionGoal( + virtual std::pair, kj::Promise> + makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt @@ -58,7 +60,8 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; + virtual std::pair> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; }; // elaborate hoax to let goals access factory methods while hiding them from the public @@ -94,13 +97,27 @@ private: */ WeakGoals awake; + template + struct CachedGoal + { + std::weak_ptr goal; + kj::Own> promise; + kj::Own> fulfiller; + + CachedGoal() + { + auto pf = kj::newPromiseAndFulfiller(); + promise = kj::heap(pf.promise.fork()); + fulfiller = std::move(pf.fulfiller); + } + }; /** * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - std::map> derivationGoals; - std::map> substitutionGoals; - std::map> drvOutputSubstitutionGoals; + std::map> derivationGoals; + std::map> substitutionGoals; + std::map> drvOutputSubstitutionGoals; /** * Cache for pathContentsGood(). @@ -226,21 +243,31 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeDerivationGoalCommon( + std::pair, kj::Promise> makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); - std::shared_ptr makeDerivationGoal( + std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; - std::shared_ptr makeBasicDerivationGoal( + std::pair, kj::Promise> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; - std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; + std::pair, kj::Promise> + makePathSubstitutionGoal( + const StorePath & storePath, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) override; + std::pair, kj::Promise> + makeDrvOutputSubstitutionGoal( + const DrvOutput & id, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) override; /** * Make a goal corresponding to the `DerivedPath`. @@ -248,13 +275,16 @@ private: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; + std::pair> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: + using Targets = std::map>; + /** * Loop until the specified top-level goals have finished. */ - Goals run(std::function req); + std::vector run(std::function req); /*** * The exit status in case of failure.