Merge pull request #8829 from obsidiansystems/build-dynamic-derivations

Adapt scheduler to work with dynamic derivations
This commit is contained in:
John Ericson 2023-08-25 11:13:15 -04:00 committed by GitHub
commit 50f40ac4c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 588 additions and 97 deletions

View file

@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V
{ {
SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
state.mkSingleDerivedPathString( state.mkSingleDerivedPathString(
SingleDerivedPath::Built { SingleDerivedPath::Built {

View file

@ -0,0 +1,157 @@
#include "create-derivation-and-realise-goal.hh"
#include "worker.hh"
namespace nix {
CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
, drvReq(drvReq)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
state = &CreateDerivationAndRealiseGoal::getDerivation;
name = fmt(
"outer obtaining drv from '%s' and then building outputs %s",
drvReq->to_string(worker.store),
std::visit(overloaded {
[&](const OutputsSpec::All) -> std::string {
return "* (all of them)";
},
[&](const OutputsSpec::Names os) {
return concatStringsSep(", ", quoteStrings(os));
},
}, wantedOutputs.raw));
trace("created outer");
worker.updateProgress();
}
CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
{
}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) {
return bo.path;
},
[&](const SingleDerivedPath::Built & bfd) {
return pathPartOfReq(*bfd.drvPath);
},
}, req.raw());
}
std::string CreateDerivationAndRealiseGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before "baboon". And
substitution goals and inner derivation goals always happen before
derivation goals (due to "b$"). */
return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
}
void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
{
}
void CreateDerivationAndRealiseGoal::work()
{
(this->*state)();
}
void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
{
/* If we already want all outputs, there is nothing to do. */
auto newWanted = wantedOutputs.union_(outputs);
bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
wantedOutputs = newWanted;
if (!needRestart) return;
if (!optDrvPath)
// haven't started steps where the outputs matter yet
return;
worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
}
void CreateDerivationAndRealiseGoal::getDerivation()
{
trace("outer init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
if (buildMode != bmNormal) return std::nullopt;
auto drvPath = StorePath::dummy;
try {
drvPath = resolveDerivedPath(worker.store, *drvReq);
} catch (MissingRealisation) {
return std::nullopt;
}
return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
? std::optional { drvPath }
: std::nullopt;
}()) {
trace(fmt("already have drv '%s' for '%s', can go straight to building",
worker.store.printStorePath(*optDrvPath),
drvReq->to_string(worker.store)));
loadAndBuildDerivation();
} else {
trace("need to obtain drv we want to build");
addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
if (waitees.empty()) work();
}
}
void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
{
trace("outer load and build derivation");
if (nrFailed != 0) {
amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
return;
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
/* Build this step! */
concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
addWaitee(upcast_goal(concreteDrvGoal));
state = &CreateDerivationAndRealiseGoal::buildDone;
optDrvPath = std::move(drvPath);
if (waitees.empty()) work();
}
void CreateDerivationAndRealiseGoal::buildDone()
{
trace("outer build done");
buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
.drvPath = drvReq,
.outputs = wantedOutputs,
});
if (buildResult.success())
amDone(ecSuccess);
else
amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
}
}

View file

@ -0,0 +1,96 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
struct DerivationGoal;
/**
* This goal type is essentially the serial composition (like function
* composition) of a goal for getting a derivation, and then a
* `DerivationGoal` using the newly-obtained derivation.
*
* In the (currently experimental) general inductive case of derivations
* that are themselves build outputs, that first goal will be *another*
* `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
* where the derivation has no provence and is just referred to by
* (content-addressed) store path, that first goal is a
* `SubstitutionGoal`.
*
* If we already have the derivation (e.g. if the evalutator has created
* the derivation locally and then instructured the store to build it),
* we can skip the first goal entirely as a small optimization.
*/
struct CreateDerivationAndRealiseGoal : public Goal
{
/**
* How to obtain a store path of the derivation to build.
*/
ref<SingleDerivedPath> drvReq;
/**
* The path of the derivation, once obtained.
**/
std::optional<StorePath> optDrvPath;
/**
* The goal for the corresponding concrete derivation.
**/
std::shared_ptr<DerivationGoal> concreteDrvGoal;
/**
* The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
GoalState state;
/**
* The final output paths of the build.
*
* - For input-addressed derivations, always the precomputed paths
*
* - For content-addressed derivations, calcuated from whatever the
* hash ends up being. (Note that fixed outputs derivations that
* produce the "wrong" output still install that data under its
* true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~CreateDerivationAndRealiseGoal();
void timedOut(Error && ex) override;
std::string key() override;
void work() override;
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/**
* The states.
*/
void getDerivation();
void loadAndBuildDerivation();
void buildDone();
JobCategory jobCategory() const override {
return JobCategory::Administration;
};
};
}

View file

@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
, buildMode(buildMode) , buildMode(buildMode)
{ {
state = &DerivationGoal::getDerivation; state = &DerivationGoal::loadDerivation;
name = fmt( name = fmt(
"building of '%s' from .drv file", "building of '%s' from .drv file",
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
} }
void DerivationGoal::getDerivation()
{
trace("init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
void DerivationGoal::loadDerivation() void DerivationGoal::loadDerivation()
{ {
trace("loading derivation"); trace("loading derivation");
@ -380,7 +362,12 @@ void DerivationGoal::gaveUpOnSubstitution()
worker.store.printStorePath(i.first)); worker.store.printStorePath(i.first));
} }
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); addWaitee(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(i.first),
.outputs = i.second,
},
buildMode == bmRepair ? bmRepair : bmNormal));
} }
/* Copy the input sources from the eval store to the build /* Copy the input sources from the eval store to the build
@ -452,7 +439,12 @@ void DerivationGoal::repairClosure()
if (drvPath2 == outputsToDrv.end()) if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else else
addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); addWaitee(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
},
bmRepair));
} }
if (waitees.empty()) { if (waitees.empty()) {
@ -1483,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
if (!useDerivation) return; if (!useDerivation) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee); auto * dg = tryGetConcreteDrvGoal(waitee);
if (!dg) return; if (!dg) return;
auto outputs = fullDrv.inputDrvs.find(dg->drvPath); auto outputs = fullDrv.inputDrvs.find(dg->drvPath);

View file

@ -50,6 +50,13 @@ struct InitialOutput {
std::optional<InitialOutputStatus> known; std::optional<InitialOutputStatus> known;
}; };
/**
* A goal for building some or all of the outputs of a derivation.
*
* The derivation must already be present, either in the store in a drv
* or in memory. If the derivation itself needs to be gotten first, a
* `CreateDerivationAndRealiseGoal` goal must be used instead.
*/
struct DerivationGoal : public Goal struct DerivationGoal : public Goal
{ {
/** /**
@ -66,8 +73,7 @@ struct DerivationGoal : public Goal
std::shared_ptr<DerivationGoal> resolvedDrvGoal; std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/** /**
* The specific outputs that we need to build. Empty means all of * The specific outputs that we need to build.
* them.
*/ */
OutputsSpec wantedOutputs; OutputsSpec wantedOutputs;
@ -229,7 +235,6 @@ struct DerivationGoal : public Goal
/** /**
* The states. * The states.
*/ */
void getDerivation();
void loadDerivation(); void loadDerivation();
void haveDerivation(); void haveDerivation();
void outputsSubstitutionTried(); void outputsSubstitutionTried();
@ -334,7 +339,9 @@ struct DerivationGoal : public Goal
StorePathSet exportReferences(const StorePathSet & storePaths); StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() override { return JobCategory::Build; }; JobCategory jobCategory() const override {
return JobCategory::Build;
};
}; };
MakeError(NotDeterministic, BuildError); MakeError(NotDeterministic, BuildError);

View file

@ -73,7 +73,9 @@ public:
void work() override; void work() override;
void handleEOF(int fd) override; void handleEOF(int fd) override;
JobCategory jobCategory() override { return JobCategory::Substitution; }; JobCategory jobCategory() const override {
return JobCategory::Substitution;
};
}; };
} }

View file

@ -1,5 +1,6 @@
#include "worker.hh" #include "worker.hh"
#include "substitution-goal.hh" #include "substitution-goal.hh"
#include "create-derivation-and-realise-goal.hh"
#include "derivation-goal.hh" #include "derivation-goal.hh"
#include "local-store.hh" #include "local-store.hh"
@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals); worker.run(goals);
StorePathSet failed; StringSet failed;
std::optional<Error> ex; std::optional<Error> ex;
for (auto & i : goals) { for (auto & i : goals) {
if (i->ex) { if (i->ex) {
@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex = std::move(i->ex); ex = std::move(i->ex);
} }
if (i->exitCode != Goal::ecSuccess) { if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath); if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath); failed.insert(i2->drvReq->to_string(*this));
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
failed.insert(printStorePath(i2->storePath));
} }
} }
@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
throw std::move(*ex); throw std::move(*ex);
} else if (!failed.empty()) { } else if (!failed.empty()) {
if (ex) logError(ex->info()); if (ex) logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
} }
} }
@ -124,8 +127,11 @@ void Store::repairPath(const StorePath & path)
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) { if (info->deriver && isValidPath(*info->deriver)) {
goals.clear(); goals.clear();
// FIXME: Should just build the specific output we need. goals.insert(worker.makeGoal(DerivedPath::Built {
goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); .drvPath = makeConstantStorePathRef(*info->deriver),
// FIXME: Should just build the specific output we need.
.outputs = OutputsSpec::All { },
}, bmRepair));
worker.run(goals); worker.run(goals);
} else } else
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));

View file

@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
} }
BuildResult Goal::getBuildResult(const DerivedPath & req) { BuildResult Goal::getBuildResult(const DerivedPath & req) const {
BuildResult res { buildResult }; BuildResult res { buildResult };
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) { if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {

View file

@ -41,8 +41,24 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
* of each category in parallel. * of each category in parallel.
*/ */
enum struct JobCategory { enum struct JobCategory {
/**
* A build of a derivation; it will use CPU and disk resources.
*/
Build, Build,
/**
* A substitution an arbitrary store object; it will use network resources.
*/
Substitution, Substitution,
/**
* A goal that does no "real" work by itself, and just exists to depend on
* other goals which *do* do real work. These goals therefore are not
* limited.
*
* These goals cannot infinitely create themselves, so there is no risk of
* a "fork bomb" type situation (which would be a problem even though the
* goal do no real work) either.
*/
Administration,
}; };
struct Goal : public std::enable_shared_from_this<Goal> struct Goal : public std::enable_shared_from_this<Goal>
@ -110,7 +126,7 @@ public:
* sake of both privacy and determinism, and this "safe accessor" * sake of both privacy and determinism, and this "safe accessor"
* ensures we don't. * ensures we don't.
*/ */
BuildResult getBuildResult(const DerivedPath &); BuildResult getBuildResult(const DerivedPath &) const;
/** /**
* Exception containing an error message, if any. * Exception containing an error message, if any.
@ -144,7 +160,7 @@ public:
void trace(std::string_view s); void trace(std::string_view s);
std::string getName() std::string getName() const
{ {
return name; return name;
} }
@ -166,7 +182,7 @@ public:
* @brief Hint for the scheduler, which concurrency limit applies. * @brief Hint for the scheduler, which concurrency limit applies.
* @see JobCategory * @see JobCategory
*/ */
virtual JobCategory jobCategory() = 0; virtual JobCategory jobCategory() const = 0;
}; };
void addToWeakGoals(WeakGoals & goals, GoalPtr p); void addToWeakGoals(WeakGoals & goals, GoalPtr p);

View file

@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd)
} }
StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
{ {
return worker.store.makeStorePath( return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),

View file

@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal
* @todo Add option to randomize, so we can audit whether our * @todo Add option to randomize, so we can audit whether our
* rewrites caught everything * rewrites caught everything
*/ */
StorePath makeFallbackPath(std::string_view outputName); StorePath makeFallbackPath(OutputNameView outputName);
}; };
} }

View file

@ -117,7 +117,9 @@ public:
/* Called by destructor, can't be overridden */ /* Called by destructor, can't be overridden */
void cleanup() override final; void cleanup() override final;
JobCategory jobCategory() override { return JobCategory::Substitution; }; JobCategory jobCategory() const override {
return JobCategory::Substitution;
};
}; };
} }

View file

@ -2,6 +2,7 @@
#include "worker.hh" #include "worker.hh"
#include "substitution-goal.hh" #include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh" #include "drv-output-substitution-goal.hh"
#include "create-derivation-and-realise-goal.hh"
#include "local-derivation-goal.hh" #include "local-derivation-goal.hh"
#include "hook-instance.hh" #include "hook-instance.hh"
@ -41,6 +42,24 @@ Worker::~Worker()
} }
std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
BuildMode buildMode)
{
std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
if (!goal) {
goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath, const StorePath & drvPath,
const OutputsSpec & wantedOutputs, const OutputsSpec & wantedOutputs,
@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr { [&](const DerivedPath::Built & bfd) -> GoalPtr {
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath)) return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
}, },
[&](const DerivedPath::Opaque & bo) -> GoalPtr { [&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
} }
template<typename K, typename V, typename F>
static void cullMap(std::map<K, V> & goalMap, F f)
{
for (auto i = goalMap.begin(); i != goalMap.end();)
if (!f(i->second))
i = goalMap.erase(i);
else ++i;
}
template<typename K, typename G> template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap) static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{ {
/* !!! inefficient */ /* !!! inefficient */
for (auto i = goalMap.begin(); cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
i != goalMap.end(); ) return gp.lock() != goal;
if (i->second.lock() == goal) { });
auto j = i; ++j; }
goalMap.erase(i);
i = j; template<typename K>
} static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
else ++i;
template<typename K>
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
{
/* !!! inefficient */
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
if (node.value.lock() == goal)
node.value.reset();
removeGoal(goal, node.childMap);
return !node.value.expired() || !node.childMap.empty();
});
} }
void Worker::removeGoal(GoalPtr goal) void Worker::removeGoal(GoalPtr goal)
{ {
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal)) if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
nix::removeGoal(drvGoal, outerDerivationGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals); nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals); nix::removeGoal(subGoal, substitutionGoals);
@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.respectTimeouts = respectTimeouts; child.respectTimeouts = respectTimeouts;
children.emplace_back(child); children.emplace_back(child);
if (inBuildSlot) { if (inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; switch (goal->jobCategory()) {
else nrLocalBuilds++; case JobCategory::Substitution:
nrSubstitutions++;
break;
case JobCategory::Build:
nrLocalBuilds++;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
}
} }
} }
@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return; if (i == children.end()) return;
if (i->inBuildSlot) { if (i->inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) { switch (goal->jobCategory()) {
case JobCategory::Substitution:
assert(nrSubstitutions > 0); assert(nrSubstitutions > 0);
nrSubstitutions--; nrSubstitutions--;
} else { break;
case JobCategory::Build:
assert(nrLocalBuilds > 0); assert(nrLocalBuilds > 0);
nrLocalBuilds--; nrLocalBuilds--;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
} }
} }
@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) { for (auto & i : _topGoals) {
topGoals.insert(i); topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) { if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built { topPaths.push_back(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(goal->drvPath), .drvPath = goal->drvReq,
.outputs = goal->wantedOutputs, .outputs = goal->wantedOutputs,
}); });
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) { } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@ -522,11 +579,26 @@ void Worker::markContentsGood(const StorePath & path)
} }
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) { GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal)
return subGoal; {
}
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
return subGoal; return subGoal;
} }
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
{
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
{
return subGoal;
}
const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
{
auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
if (!odg) return nullptr;
return &*odg->concreteDrvGoal;
}
} }

View file

@ -4,6 +4,7 @@
#include "types.hh" #include "types.hh"
#include "lock.hh" #include "lock.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derived-path-map.hh"
#include "goal.hh" #include "goal.hh"
#include "realisation.hh" #include "realisation.hh"
@ -13,6 +14,7 @@
namespace nix { namespace nix {
/* Forward definition. */ /* Forward definition. */
struct CreateDerivationAndRealiseGoal;
struct DerivationGoal; struct DerivationGoal;
struct PathSubstitutionGoal; struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal; class DrvOutputSubstitutionGoal;
@ -31,9 +33,23 @@ class DrvOutputSubstitutionGoal;
*/ */
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/**
* The current implementation of impure derivations has
* `DerivationGoal`s accumulate realisations from their waitees.
* Unfortunately, `DerivationGoal`s don't directly depend on other
* goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
*
* We try not to share any of the details of any goal type with any
* other, for sake of modularity and quicker rebuilds. This means we
* cannot "just" downcast and fish out the field. So as an escape hatch,
* we have made the function, written in `worker.cc` where all the goal
* types are visible, and use it instead.
*/
const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
/** /**
* A mapping used to remember for each child process to what goal it * A mapping used to remember for each child process to what goal it
@ -102,6 +118,9 @@ private:
* Maps used to prevent multiple instantiations of a goal for the * Maps used to prevent multiple instantiations of a goal for the
* same derivation / path. * same derivation / path.
*/ */
DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@ -189,6 +208,9 @@ public:
* @ref DerivationGoal "derivation goal" * @ref DerivationGoal "derivation goal"
*/ */
private: private:
std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvPath,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs, const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

View file

@ -12,7 +12,7 @@
namespace nix { namespace nix {
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> { [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@ -36,7 +36,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
} }
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
{ {
return store.makeFixedOutputPathFromCA( return store.makeFixedOutputPathFromCA(
outputPathName(drvName, outputName), outputPathName(drvName, outputName),
@ -466,7 +466,7 @@ bool isDerivation(std::string_view fileName)
} }
std::string outputPathName(std::string_view drvName, std::string_view outputName) { std::string outputPathName(std::string_view drvName, OutputNameView outputName) {
std::string res { drvName }; std::string res { drvName };
if (outputName != "out") { if (outputName != "out") {
res += "-"; res += "-";
@ -810,7 +810,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
} }
std::string hashPlaceholder(const std::string_view outputName) std::string hashPlaceholder(const OutputNameView outputName)
{ {
// FIXME: memoize? // FIXME: memoize?
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
@ -963,7 +963,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
const Hash impureOutputHash = hashString(htSHA256, "impure"); const Hash impureOutputHash = hashString(htSHA256, "impure");
nlohmann::json DerivationOutput::toJSON( nlohmann::json DerivationOutput::toJSON(
const Store & store, std::string_view drvName, std::string_view outputName) const const Store & store, std::string_view drvName, OutputNameView outputName) const
{ {
nlohmann::json res = nlohmann::json::object(); nlohmann::json res = nlohmann::json::object();
std::visit(overloaded { std::visit(overloaded {
@ -990,7 +990,7 @@ nlohmann::json DerivationOutput::toJSON(
DerivationOutput DerivationOutput::fromJSON( DerivationOutput DerivationOutput::fromJSON(
const Store & store, std::string_view drvName, std::string_view outputName, const Store & store, std::string_view drvName, OutputNameView outputName,
const nlohmann::json & _json, const nlohmann::json & _json,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {

View file

@ -55,7 +55,7 @@ struct DerivationOutput
* @param drvName The name of the derivation this is an output of, without the `.drv`. * @param drvName The name of the derivation this is an output of, without the `.drv`.
* @param outputName The name of this output. * @param outputName The name of this output.
*/ */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
GENERATE_CMP(CAFixed, me->ca); GENERATE_CMP(CAFixed, me->ca);
}; };
@ -132,19 +132,19 @@ struct DerivationOutput
* the safer interface provided by * the safer interface provided by
* BasicDerivation::outputsAndOptPaths * BasicDerivation::outputsAndOptPaths
*/ */
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
nlohmann::json toJSON( nlohmann::json toJSON(
const Store & store, const Store & store,
std::string_view drvName, std::string_view drvName,
std::string_view outputName) const; OutputNameView outputName) const;
/** /**
* @param xpSettings Stop-gap to avoid globals during unit tests. * @param xpSettings Stop-gap to avoid globals during unit tests.
*/ */
static DerivationOutput fromJSON( static DerivationOutput fromJSON(
const Store & store, const Store & store,
std::string_view drvName, std::string_view drvName,
std::string_view outputName, OutputNameView outputName,
const nlohmann::json & json, const nlohmann::json & json,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
}; };
@ -405,7 +405,7 @@ bool isDerivation(std::string_view fileName);
* This is usually <drv-name>-<output-name>, but is just <drv-name> when * This is usually <drv-name>-<output-name>, but is just <drv-name> when
* the output name is "out". * the output name is "out".
*/ */
std::string outputPathName(std::string_view drvName, std::string_view outputName); std::string outputPathName(std::string_view drvName, OutputNameView outputName);
/** /**
@ -499,7 +499,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
* own outputs without needing to use the hash of a derivation in * own outputs without needing to use the hash of a derivation in
* itself, making the hash near-impossible to calculate. * itself, making the hash near-impossible to calculate.
*/ */
std::string hashPlaceholder(const std::string_view outputName); std::string hashPlaceholder(const OutputNameView outputName);
extern const Hash impureOutputHash; extern const Hash impureOutputHash;

View file

@ -0,0 +1,33 @@
#include "derived-path-map.hh"
namespace nix {
template<typename V>
typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const SingleDerivedPath & k)
{
std::function<ChildNode &(const SingleDerivedPath & )> initIter;
initIter = [&](const auto & k) -> auto & {
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) -> auto & {
// will not overwrite if already there
return map[bo.path];
},
[&](const SingleDerivedPath::Built & bfd) -> auto & {
auto & n = initIter(*bfd.drvPath);
return n.childMap[bfd.output];
},
}, k.raw());
};
return initIter(k);
}
}
// instantiations
#include "create-derivation-and-realise-goal.hh"
namespace nix {
template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
}

View file

@ -0,0 +1,73 @@
#pragma once
#include "types.hh"
#include "derived-path.hh"
namespace nix {
/**
* A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to
* values.
*
* Concretely, an n-ary tree, as described below. A
* `SingleDerivedPath::Opaque` maps to the value of an immediate child
* of the root node. A `SingleDerivedPath::Built` maps to a deeper child
* node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a
* child node (inductively), and then the
* `SingleDerivedPath::Built::output` is used to look up that child's
* child via its map. In this manner, every `SingleDerivedPath` is
* mapped to a child node.
*
* @param V A type to instantiate for each output. It should probably
* should be an "optional" type so not every interior node has to have a
* value. For example, the scheduler uses
* `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
* remember which goals correspond to which outputs. `* const Something`
* or `std::optional<Something>` would also be good choices for
* "optional" types.
*/
template<typename V>
struct DerivedPathMap {
/**
* A child node (non-root node).
*/
struct ChildNode {
/**
* Value of this child node.
*
* @see DerivedPathMap for what `V` should be.
*/
V value;
/**
* The map type for the root node.
*/
using Map = std::map<OutputName, ChildNode>;
/**
* The map of the root node.
*/
Map childMap;
};
/**
* The map type for the root node.
*/
using Map = std::map<StorePath, ChildNode>;
/**
* The map of root node.
*/
Map map;
/**
* Find the node for `k`, creating it if needed.
*
* The node is referred to as a "slot" on the assumption that `V` is
* some sort of optional type, so the given key can be set or unset
* by changing this node.
*/
ChildNode & ensureSlot(const SingleDerivedPath & k);
};
}

View file

@ -167,7 +167,7 @@ void drvRequireExperiment(
SingleDerivedPath::Built SingleDerivedPath::Built::parse( SingleDerivedPath::Built SingleDerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv, const Store & store, ref<SingleDerivedPath> drv,
std::string_view output, OutputNameView output,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
drvRequireExperiment(*drv, xpSettings); drvRequireExperiment(*drv, xpSettings);
@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
DerivedPath::Built DerivedPath::Built::parse( DerivedPath::Built DerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv, const Store & store, ref<SingleDerivedPath> drv,
std::string_view outputsS, OutputNameView outputsS,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
drvRequireExperiment(*drv, xpSettings); drvRequireExperiment(*drv, xpSettings);

View file

@ -42,7 +42,7 @@ struct SingleDerivedPath;
*/ */
struct SingleDerivedPathBuilt { struct SingleDerivedPathBuilt {
ref<SingleDerivedPath> drvPath; ref<SingleDerivedPath> drvPath;
std::string output; OutputName output;
/** /**
* Get the store path this is ultimately derived from (by realising * Get the store path this is ultimately derived from (by realising
@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt {
*/ */
static SingleDerivedPathBuilt parse( static SingleDerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath> drvPath, const Store & store, ref<SingleDerivedPath> drvPath,
std::string_view outputs, OutputNameView outputs,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const; nlohmann::json toJSON(Store & store) const;

View file

@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
const StorePath & drvPath, const StorePath & drvPath,
std::string_view outputName, OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
xpSettings.require(Xp::CaDerivations); xpSettings.require(Xp::CaDerivations);
@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
const DownstreamPlaceholder & placeholder, const DownstreamPlaceholder & placeholder,
std::string_view outputName, OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
xpSettings.require(Xp::DynamicDerivations); xpSettings.require(Xp::DynamicDerivations);

View file

@ -58,7 +58,7 @@ public:
*/ */
static DownstreamPlaceholder unknownCaOutput( static DownstreamPlaceholder unknownCaOutput(
const StorePath & drvPath, const StorePath & drvPath,
std::string_view outputName, OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/** /**
@ -72,7 +72,7 @@ public:
*/ */
static DownstreamPlaceholder unknownDerivation( static DownstreamPlaceholder unknownDerivation(
const DownstreamPlaceholder & drvPlaceholder, const DownstreamPlaceholder & drvPlaceholder,
std::string_view outputName, OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/** /**

View file

@ -399,8 +399,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
store.printStorePath(drvPath), bfd.output); store.printStorePath(drvPath), bfd.output);
auto & optPath = outputPaths.at(bfd.output); auto & optPath = outputPaths.at(bfd.output);
if (!optPath) if (!optPath)
throw Error("'%s' does not yet map to a known concrete store path", throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
bfd.to_string(store));
return *optPath; return *optPath;
}, },
}, req.raw()); }, req.raw());

View file

@ -13,24 +13,36 @@
namespace nix { namespace nix {
/**
* An (owned) output name. Just a type alias used to make code more
* readible.
*/
typedef std::string OutputName;
/**
* A borrowed output name. Just a type alias used to make code more
* readible.
*/
typedef std::string_view OutputNameView;
struct OutputsSpec { struct OutputsSpec {
/** /**
* A non-empty set of outputs, specified by name * A non-empty set of outputs, specified by name
*/ */
struct Names : std::set<std::string> { struct Names : std::set<OutputName> {
using std::set<std::string>::set; using std::set<OutputName>::set;
/* These need to be "inherited manually" */ /* These need to be "inherited manually" */
Names(const std::set<std::string> & s) Names(const std::set<OutputName> & s)
: std::set<std::string>(s) : std::set<OutputName>(s)
{ assert(!empty()); } { assert(!empty()); }
/** /**
* Needs to be "inherited manually" * Needs to be "inherited manually"
*/ */
Names(std::set<std::string> && s) Names(std::set<OutputName> && s)
: std::set<std::string>(s) : std::set<OutputName>(s)
{ assert(!empty()); } { assert(!empty()); }
/* This set should always be non-empty, so we delete this /* This set should always be non-empty, so we delete this
@ -57,7 +69,7 @@ struct OutputsSpec {
*/ */
OutputsSpec() = delete; OutputsSpec() = delete;
bool contains(const std::string & output) const; bool contains(const OutputName & output) const;
/** /**
* Create a new OutputsSpec which is the union of this and that. * Create a new OutputsSpec which is the union of this and that.

View file

@ -34,7 +34,7 @@ struct DrvOutput {
/** /**
* The name of the output. * The name of the output.
*/ */
std::string outputName; OutputName outputName;
std::string to_string() const; std::string to_string() const;
@ -84,7 +84,7 @@ struct Realisation {
* Since these are the outputs of a single derivation, we know the * Since these are the outputs of a single derivation, we know the
* output names are unique so we can use them as the map key. * output names are unique so we can use them as the map key.
*/ */
typedef std::map<std::string, Realisation> SingleDrvOutputs; typedef std::map<OutputName, Realisation> SingleDrvOutputs;
/** /**
* Collection type for multiple derivations' outputs' `Realisation`s. * Collection type for multiple derivations' outputs' `Realisation`s.
@ -146,7 +146,7 @@ public:
MissingRealisation(DrvOutput & outputId) MissingRealisation(DrvOutput & outputId)
: MissingRealisation(outputId.outputName, outputId.strHash()) : MissingRealisation(outputId.outputName, outputId.strHash())
{} {}
MissingRealisation(std::string_view drv, std::string outputName) MissingRealisation(std::string_view drv, OutputName outputName)
: Error( "cannot operate on output '%s' of the " : Error( "cannot operate on output '%s' of the "
"unbuilt derivation '%s'", "unbuilt derivation '%s'",
outputName, outputName,

View file

@ -18,4 +18,6 @@ clearStore
drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" out2=$(nix build "${drvDep}^out^out" --no-link)
test $out1 == $out2