Merge pull request #9086 from NixOS/backport-9081-to-2.18-maintenance
[Backport 2.18-maintenance] Revert "Adapt scheduler to work with dynamic derivations"
This commit is contained in:
commit
f5f4de6a55
|
@ -1,157 +0,0 @@
|
||||||
#include "create-derivation-and-realise-goal.hh"
|
|
||||||
#include "worker.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
|
||||||
|
|
||||||
CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
|
|
||||||
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
|
||||||
: Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
|
|
||||||
, drvReq(drvReq)
|
|
||||||
, wantedOutputs(wantedOutputs)
|
|
||||||
, buildMode(buildMode)
|
|
||||||
{
|
|
||||||
state = &CreateDerivationAndRealiseGoal::getDerivation;
|
|
||||||
name = fmt(
|
|
||||||
"outer obtaining drv from '%s' and then building outputs %s",
|
|
||||||
drvReq->to_string(worker.store),
|
|
||||||
std::visit(overloaded {
|
|
||||||
[&](const OutputsSpec::All) -> std::string {
|
|
||||||
return "* (all of them)";
|
|
||||||
},
|
|
||||||
[&](const OutputsSpec::Names os) {
|
|
||||||
return concatStringsSep(", ", quoteStrings(os));
|
|
||||||
},
|
|
||||||
}, wantedOutputs.raw));
|
|
||||||
trace("created outer");
|
|
||||||
|
|
||||||
worker.updateProgress();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static StorePath pathPartOfReq(const SingleDerivedPath & req)
|
|
||||||
{
|
|
||||||
return std::visit(overloaded {
|
|
||||||
[&](const SingleDerivedPath::Opaque & bo) {
|
|
||||||
return bo.path;
|
|
||||||
},
|
|
||||||
[&](const SingleDerivedPath::Built & bfd) {
|
|
||||||
return pathPartOfReq(*bfd.drvPath);
|
|
||||||
},
|
|
||||||
}, req.raw());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
std::string CreateDerivationAndRealiseGoal::key()
|
|
||||||
{
|
|
||||||
/* Ensure that derivations get built in order of their name,
|
|
||||||
i.e. a derivation named "aardvark" always comes before "baboon". And
|
|
||||||
substitution goals and inner derivation goals always happen before
|
|
||||||
derivation goals (due to "b$"). */
|
|
||||||
return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::work()
|
|
||||||
{
|
|
||||||
(this->*state)();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
|
|
||||||
{
|
|
||||||
/* If we already want all outputs, there is nothing to do. */
|
|
||||||
auto newWanted = wantedOutputs.union_(outputs);
|
|
||||||
bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
|
|
||||||
wantedOutputs = newWanted;
|
|
||||||
|
|
||||||
if (!needRestart) return;
|
|
||||||
|
|
||||||
if (!optDrvPath)
|
|
||||||
// haven't started steps where the outputs matter yet
|
|
||||||
return;
|
|
||||||
worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::getDerivation()
|
|
||||||
{
|
|
||||||
trace("outer init");
|
|
||||||
|
|
||||||
/* The first thing to do is to make sure that the derivation
|
|
||||||
exists. If it doesn't, it may be created through a
|
|
||||||
substitute. */
|
|
||||||
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
|
|
||||||
if (buildMode != bmNormal) return std::nullopt;
|
|
||||||
|
|
||||||
auto drvPath = StorePath::dummy;
|
|
||||||
try {
|
|
||||||
drvPath = resolveDerivedPath(worker.store, *drvReq);
|
|
||||||
} catch (MissingRealisation &) {
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
|
|
||||||
? std::optional { drvPath }
|
|
||||||
: std::nullopt;
|
|
||||||
}()) {
|
|
||||||
trace(fmt("already have drv '%s' for '%s', can go straight to building",
|
|
||||||
worker.store.printStorePath(*optDrvPath),
|
|
||||||
drvReq->to_string(worker.store)));
|
|
||||||
|
|
||||||
loadAndBuildDerivation();
|
|
||||||
} else {
|
|
||||||
trace("need to obtain drv we want to build");
|
|
||||||
|
|
||||||
addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
|
|
||||||
|
|
||||||
state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
|
|
||||||
if (waitees.empty()) work();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
|
|
||||||
{
|
|
||||||
trace("outer load and build derivation");
|
|
||||||
|
|
||||||
if (nrFailed != 0) {
|
|
||||||
amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
|
||||||
/* Build this step! */
|
|
||||||
concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
|
|
||||||
addWaitee(upcast_goal(concreteDrvGoal));
|
|
||||||
state = &CreateDerivationAndRealiseGoal::buildDone;
|
|
||||||
optDrvPath = std::move(drvPath);
|
|
||||||
if (waitees.empty()) work();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void CreateDerivationAndRealiseGoal::buildDone()
|
|
||||||
{
|
|
||||||
trace("outer build done");
|
|
||||||
|
|
||||||
buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
|
|
||||||
.drvPath = drvReq,
|
|
||||||
.outputs = wantedOutputs,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (buildResult.success())
|
|
||||||
amDone(ecSuccess);
|
|
||||||
else
|
|
||||||
amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,96 +0,0 @@
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include "parsed-derivations.hh"
|
|
||||||
#include "lock.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
#include "goal.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
|
||||||
|
|
||||||
struct DerivationGoal;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This goal type is essentially the serial composition (like function
|
|
||||||
* composition) of a goal for getting a derivation, and then a
|
|
||||||
* `DerivationGoal` using the newly-obtained derivation.
|
|
||||||
*
|
|
||||||
* In the (currently experimental) general inductive case of derivations
|
|
||||||
* that are themselves build outputs, that first goal will be *another*
|
|
||||||
* `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
|
|
||||||
* where the derivation has no provence and is just referred to by
|
|
||||||
* (content-addressed) store path, that first goal is a
|
|
||||||
* `SubstitutionGoal`.
|
|
||||||
*
|
|
||||||
* If we already have the derivation (e.g. if the evalutator has created
|
|
||||||
* the derivation locally and then instructured the store to build it),
|
|
||||||
* we can skip the first goal entirely as a small optimization.
|
|
||||||
*/
|
|
||||||
struct CreateDerivationAndRealiseGoal : public Goal
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* How to obtain a store path of the derivation to build.
|
|
||||||
*/
|
|
||||||
ref<SingleDerivedPath> drvReq;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The path of the derivation, once obtained.
|
|
||||||
**/
|
|
||||||
std::optional<StorePath> optDrvPath;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The goal for the corresponding concrete derivation.
|
|
||||||
**/
|
|
||||||
std::shared_ptr<DerivationGoal> concreteDrvGoal;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The specific outputs that we need to build.
|
|
||||||
*/
|
|
||||||
OutputsSpec wantedOutputs;
|
|
||||||
|
|
||||||
typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
|
|
||||||
GoalState state;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The final output paths of the build.
|
|
||||||
*
|
|
||||||
* - For input-addressed derivations, always the precomputed paths
|
|
||||||
*
|
|
||||||
* - For content-addressed derivations, calcuated from whatever the
|
|
||||||
* hash ends up being. (Note that fixed outputs derivations that
|
|
||||||
* produce the "wrong" output still install that data under its
|
|
||||||
* true content-address.)
|
|
||||||
*/
|
|
||||||
OutputPathMap finalOutputs;
|
|
||||||
|
|
||||||
BuildMode buildMode;
|
|
||||||
|
|
||||||
CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
|
|
||||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
|
||||||
BuildMode buildMode = bmNormal);
|
|
||||||
virtual ~CreateDerivationAndRealiseGoal();
|
|
||||||
|
|
||||||
void timedOut(Error && ex) override;
|
|
||||||
|
|
||||||
std::string key() override;
|
|
||||||
|
|
||||||
void work() override;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add wanted outputs to an already existing derivation goal.
|
|
||||||
*/
|
|
||||||
void addWantedOutputs(const OutputsSpec & outputs);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The states.
|
|
||||||
*/
|
|
||||||
void getDerivation();
|
|
||||||
void loadAndBuildDerivation();
|
|
||||||
void buildDone();
|
|
||||||
|
|
||||||
JobCategory jobCategory() const override {
|
|
||||||
return JobCategory::Administration;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
|
@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
||||||
, wantedOutputs(wantedOutputs)
|
, wantedOutputs(wantedOutputs)
|
||||||
, buildMode(buildMode)
|
, buildMode(buildMode)
|
||||||
{
|
{
|
||||||
state = &DerivationGoal::loadDerivation;
|
state = &DerivationGoal::getDerivation;
|
||||||
name = fmt(
|
name = fmt(
|
||||||
"building of '%s' from .drv file",
|
"building of '%s' from .drv file",
|
||||||
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
|
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
|
||||||
|
@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void DerivationGoal::getDerivation()
|
||||||
|
{
|
||||||
|
trace("init");
|
||||||
|
|
||||||
|
/* The first thing to do is to make sure that the derivation
|
||||||
|
exists. If it doesn't, it may be created through a
|
||||||
|
substitute. */
|
||||||
|
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
|
||||||
|
loadDerivation();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
|
||||||
|
|
||||||
|
state = &DerivationGoal::loadDerivation;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void DerivationGoal::loadDerivation()
|
void DerivationGoal::loadDerivation()
|
||||||
{
|
{
|
||||||
trace("loading derivation");
|
trace("loading derivation");
|
||||||
|
@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||||
if (!useDerivation) return;
|
if (!useDerivation) return;
|
||||||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
||||||
|
|
||||||
std::optional info = tryGetConcreteDrvGoal(waitee);
|
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
|
||||||
if (!info) return;
|
if (!dg) return;
|
||||||
const auto & [dg, drvReq] = *info;
|
|
||||||
|
|
||||||
auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
|
auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
|
||||||
if (!nodeP) return;
|
if (!nodeP) return;
|
||||||
auto & outputs = nodeP->value;
|
auto & outputs = nodeP->value;
|
||||||
|
|
||||||
for (auto & outputName : outputs) {
|
for (auto & outputName : outputs) {
|
||||||
auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
|
auto buildResult = dg->getBuildResult(DerivedPath::Built {
|
||||||
.drvPath = makeConstantStorePathRef(dg.get().drvPath),
|
.drvPath = makeConstantStorePathRef(dg->drvPath),
|
||||||
.outputs = OutputsSpec::Names { outputName },
|
.outputs = OutputsSpec::Names { outputName },
|
||||||
});
|
});
|
||||||
if (buildResult.success()) {
|
if (buildResult.success()) {
|
||||||
auto i = buildResult.builtOutputs.find(outputName);
|
auto i = buildResult.builtOutputs.find(outputName);
|
||||||
if (i != buildResult.builtOutputs.end())
|
if (i != buildResult.builtOutputs.end())
|
||||||
inputDrvOutputs.insert_or_assign(
|
inputDrvOutputs.insert_or_assign(
|
||||||
{ dg.get().drvPath, outputName },
|
{ dg->drvPath, outputName },
|
||||||
i->second.outPath);
|
i->second.outPath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -52,10 +52,6 @@ struct InitialOutput {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A goal for building some or all of the outputs of a derivation.
|
* A goal for building some or all of the outputs of a derivation.
|
||||||
*
|
|
||||||
* The derivation must already be present, either in the store in a drv
|
|
||||||
* or in memory. If the derivation itself needs to be gotten first, a
|
|
||||||
* `CreateDerivationAndRealiseGoal` goal must be used instead.
|
|
||||||
*/
|
*/
|
||||||
struct DerivationGoal : public Goal
|
struct DerivationGoal : public Goal
|
||||||
{
|
{
|
||||||
|
@ -235,6 +231,7 @@ struct DerivationGoal : public Goal
|
||||||
/**
|
/**
|
||||||
* The states.
|
* The states.
|
||||||
*/
|
*/
|
||||||
|
void getDerivation();
|
||||||
void loadDerivation();
|
void loadDerivation();
|
||||||
void haveDerivation();
|
void haveDerivation();
|
||||||
void outputsSubstitutionTried();
|
void outputsSubstitutionTried();
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
#include "worker.hh"
|
#include "worker.hh"
|
||||||
#include "substitution-goal.hh"
|
#include "substitution-goal.hh"
|
||||||
#include "create-derivation-and-realise-goal.hh"
|
|
||||||
#include "derivation-goal.hh"
|
#include "derivation-goal.hh"
|
||||||
#include "local-store.hh"
|
#include "local-store.hh"
|
||||||
|
|
||||||
|
@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
||||||
|
|
||||||
worker.run(goals);
|
worker.run(goals);
|
||||||
|
|
||||||
StringSet failed;
|
StorePathSet failed;
|
||||||
std::optional<Error> ex;
|
std::optional<Error> ex;
|
||||||
for (auto & i : goals) {
|
for (auto & i : goals) {
|
||||||
if (i->ex) {
|
if (i->ex) {
|
||||||
|
@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
||||||
ex = std::move(i->ex);
|
ex = std::move(i->ex);
|
||||||
}
|
}
|
||||||
if (i->exitCode != Goal::ecSuccess) {
|
if (i->exitCode != Goal::ecSuccess) {
|
||||||
if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
|
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
|
||||||
failed.insert(i2->drvReq->to_string(*this));
|
failed.insert(i2->drvPath);
|
||||||
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
||||||
failed.insert(printStorePath(i2->storePath));
|
failed.insert(i2->storePath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
||||||
throw std::move(*ex);
|
throw std::move(*ex);
|
||||||
} else if (!failed.empty()) {
|
} else if (!failed.empty()) {
|
||||||
if (ex) logError(ex->info());
|
if (ex) logError(ex->info());
|
||||||
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
|
throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -49,16 +49,6 @@ enum struct JobCategory {
|
||||||
* A substitution an arbitrary store object; it will use network resources.
|
* A substitution an arbitrary store object; it will use network resources.
|
||||||
*/
|
*/
|
||||||
Substitution,
|
Substitution,
|
||||||
/**
|
|
||||||
* A goal that does no "real" work by itself, and just exists to depend on
|
|
||||||
* other goals which *do* do real work. These goals therefore are not
|
|
||||||
* limited.
|
|
||||||
*
|
|
||||||
* These goals cannot infinitely create themselves, so there is no risk of
|
|
||||||
* a "fork bomb" type situation (which would be a problem even though the
|
|
||||||
* goal do no real work) either.
|
|
||||||
*/
|
|
||||||
Administration,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Goal : public std::enable_shared_from_this<Goal>
|
struct Goal : public std::enable_shared_from_this<Goal>
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
#include "worker.hh"
|
#include "worker.hh"
|
||||||
#include "substitution-goal.hh"
|
#include "substitution-goal.hh"
|
||||||
#include "drv-output-substitution-goal.hh"
|
#include "drv-output-substitution-goal.hh"
|
||||||
#include "create-derivation-and-realise-goal.hh"
|
|
||||||
#include "local-derivation-goal.hh"
|
#include "local-derivation-goal.hh"
|
||||||
#include "hook-instance.hh"
|
#include "hook-instance.hh"
|
||||||
|
|
||||||
|
@ -42,24 +41,6 @@ Worker::~Worker()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
|
|
||||||
ref<SingleDerivedPath> drvReq,
|
|
||||||
const OutputsSpec & wantedOutputs,
|
|
||||||
BuildMode buildMode)
|
|
||||||
{
|
|
||||||
std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
|
|
||||||
std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
|
|
||||||
if (!goal) {
|
|
||||||
goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
|
|
||||||
goal_weak = goal;
|
|
||||||
wakeUp(goal);
|
|
||||||
} else {
|
|
||||||
goal->addWantedOutputs(wantedOutputs);
|
|
||||||
}
|
|
||||||
return goal;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
||||||
const StorePath & drvPath,
|
const StorePath & drvPath,
|
||||||
const OutputsSpec & wantedOutputs,
|
const OutputsSpec & wantedOutputs,
|
||||||
|
@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
|
||||||
{
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[&](const DerivedPath::Built & bfd) -> GoalPtr {
|
[&](const DerivedPath::Built & bfd) -> GoalPtr {
|
||||||
return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
|
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
|
||||||
|
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
|
||||||
|
else
|
||||||
|
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
|
||||||
},
|
},
|
||||||
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
|
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
|
||||||
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
|
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
|
||||||
|
@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
template<typename K, typename V, typename F>
|
|
||||||
static void cullMap(std::map<K, V> & goalMap, F f)
|
|
||||||
{
|
|
||||||
for (auto i = goalMap.begin(); i != goalMap.end();)
|
|
||||||
if (!f(i->second))
|
|
||||||
i = goalMap.erase(i);
|
|
||||||
else ++i;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<typename K, typename G>
|
template<typename K, typename G>
|
||||||
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
|
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
|
||||||
{
|
{
|
||||||
/* !!! inefficient */
|
/* !!! inefficient */
|
||||||
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
|
for (auto i = goalMap.begin();
|
||||||
return gp.lock() != goal;
|
i != goalMap.end(); )
|
||||||
});
|
if (i->second.lock() == goal) {
|
||||||
}
|
auto j = i; ++j;
|
||||||
|
goalMap.erase(i);
|
||||||
template<typename K>
|
i = j;
|
||||||
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
|
}
|
||||||
|
else ++i;
|
||||||
template<typename K>
|
|
||||||
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
|
|
||||||
{
|
|
||||||
/* !!! inefficient */
|
|
||||||
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
|
|
||||||
if (node.value.lock() == goal)
|
|
||||||
node.value.reset();
|
|
||||||
removeGoal(goal, node.childMap);
|
|
||||||
return !node.value.expired() || !node.childMap.empty();
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::removeGoal(GoalPtr goal)
|
void Worker::removeGoal(GoalPtr goal)
|
||||||
{
|
{
|
||||||
if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
|
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||||
nix::removeGoal(drvGoal, outerDerivationGoals.map);
|
|
||||||
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
|
||||||
nix::removeGoal(drvGoal, derivationGoals);
|
nix::removeGoal(drvGoal, derivationGoals);
|
||||||
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||||
nix::removeGoal(subGoal, substitutionGoals);
|
nix::removeGoal(subGoal, substitutionGoals);
|
||||||
|
@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||||
child.respectTimeouts = respectTimeouts;
|
child.respectTimeouts = respectTimeouts;
|
||||||
children.emplace_back(child);
|
children.emplace_back(child);
|
||||||
if (inBuildSlot) {
|
if (inBuildSlot) {
|
||||||
switch (goal->jobCategory()) {
|
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
|
||||||
case JobCategory::Substitution:
|
else nrLocalBuilds++;
|
||||||
nrSubstitutions++;
|
|
||||||
break;
|
|
||||||
case JobCategory::Build:
|
|
||||||
nrLocalBuilds++;
|
|
||||||
break;
|
|
||||||
case JobCategory::Administration:
|
|
||||||
/* Intentionally not limited, see docs */
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
abort();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
||||||
if (i == children.end()) return;
|
if (i == children.end()) return;
|
||||||
|
|
||||||
if (i->inBuildSlot) {
|
if (i->inBuildSlot) {
|
||||||
switch (goal->jobCategory()) {
|
if (goal->jobCategory() == JobCategory::Substitution) {
|
||||||
case JobCategory::Substitution:
|
|
||||||
assert(nrSubstitutions > 0);
|
assert(nrSubstitutions > 0);
|
||||||
nrSubstitutions--;
|
nrSubstitutions--;
|
||||||
break;
|
} else {
|
||||||
case JobCategory::Build:
|
|
||||||
assert(nrLocalBuilds > 0);
|
assert(nrLocalBuilds > 0);
|
||||||
nrLocalBuilds--;
|
nrLocalBuilds--;
|
||||||
break;
|
|
||||||
case JobCategory::Administration:
|
|
||||||
/* Intentionally not limited, see docs */
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
abort();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals)
|
||||||
|
|
||||||
for (auto & i : _topGoals) {
|
for (auto & i : _topGoals) {
|
||||||
topGoals.insert(i);
|
topGoals.insert(i);
|
||||||
if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
|
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
|
||||||
topPaths.push_back(DerivedPath::Built {
|
topPaths.push_back(DerivedPath::Built {
|
||||||
.drvPath = goal->drvReq,
|
.drvPath = makeConstantStorePathRef(goal->drvPath),
|
||||||
.outputs = goal->wantedOutputs,
|
.outputs = goal->wantedOutputs,
|
||||||
});
|
});
|
||||||
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||||
|
@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
|
||||||
return subGoal;
|
return subGoal;
|
||||||
}
|
}
|
||||||
|
|
||||||
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
|
|
||||||
{
|
|
||||||
return subGoal;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
|
|
||||||
{
|
|
||||||
auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
|
|
||||||
if (!odg) return std::nullopt;
|
|
||||||
return {{
|
|
||||||
std::cref(*odg->concreteDrvGoal),
|
|
||||||
std::cref(*odg->drvReq),
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
#include "lock.hh"
|
#include "lock.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "derived-path-map.hh"
|
|
||||||
#include "goal.hh"
|
#include "goal.hh"
|
||||||
#include "realisation.hh"
|
#include "realisation.hh"
|
||||||
|
|
||||||
|
@ -14,7 +13,6 @@
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
/* Forward definition. */
|
/* Forward definition. */
|
||||||
struct CreateDerivationAndRealiseGoal;
|
|
||||||
struct DerivationGoal;
|
struct DerivationGoal;
|
||||||
struct PathSubstitutionGoal;
|
struct PathSubstitutionGoal;
|
||||||
class DrvOutputSubstitutionGoal;
|
class DrvOutputSubstitutionGoal;
|
||||||
|
@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal;
|
||||||
*/
|
*/
|
||||||
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
|
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
|
||||||
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
|
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
|
||||||
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
|
|
||||||
|
|
||||||
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
||||||
|
|
||||||
/**
|
|
||||||
* The current implementation of impure derivations has
|
|
||||||
* `DerivationGoal`s accumulate realisations from their waitees.
|
|
||||||
* Unfortunately, `DerivationGoal`s don't directly depend on other
|
|
||||||
* goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
|
|
||||||
*
|
|
||||||
* We try not to share any of the details of any goal type with any
|
|
||||||
* other, for sake of modularity and quicker rebuilds. This means we
|
|
||||||
* cannot "just" downcast and fish out the field. So as an escape hatch,
|
|
||||||
* we have made the function, written in `worker.cc` where all the goal
|
|
||||||
* types are visible, and use it instead.
|
|
||||||
*/
|
|
||||||
|
|
||||||
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A mapping used to remember for each child process to what goal it
|
* A mapping used to remember for each child process to what goal it
|
||||||
* belongs, and file descriptors for receiving log data and output
|
* belongs, and file descriptors for receiving log data and output
|
||||||
|
@ -119,9 +101,6 @@ private:
|
||||||
* Maps used to prevent multiple instantiations of a goal for the
|
* Maps used to prevent multiple instantiations of a goal for the
|
||||||
* same derivation / path.
|
* same derivation / path.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
|
|
||||||
|
|
||||||
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
|
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
|
||||||
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
|
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
|
||||||
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
|
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
|
||||||
|
@ -209,9 +188,6 @@ public:
|
||||||
* @ref DerivationGoal "derivation goal"
|
* @ref DerivationGoal "derivation goal"
|
||||||
*/
|
*/
|
||||||
private:
|
private:
|
||||||
std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
|
|
||||||
ref<SingleDerivedPath> drvPath,
|
|
||||||
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
|
|
||||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||||
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
|
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
|
||||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||||
|
|
|
@ -51,11 +51,8 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
|
||||||
|
|
||||||
// instantiations
|
// instantiations
|
||||||
|
|
||||||
#include "create-derivation-and-realise-goal.hh"
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
|
|
||||||
|
|
||||||
GENERATE_CMP_EXT(
|
GENERATE_CMP_EXT(
|
||||||
template<>,
|
template<>,
|
||||||
DerivedPathMap<std::set<std::string>>::ChildNode,
|
DerivedPathMap<std::set<std::string>>::ChildNode,
|
||||||
|
|
|
@ -20,11 +20,8 @@ namespace nix {
|
||||||
*
|
*
|
||||||
* @param V A type to instantiate for each output. It should probably
|
* @param V A type to instantiate for each output. It should probably
|
||||||
* should be an "optional" type so not every interior node has to have a
|
* should be an "optional" type so not every interior node has to have a
|
||||||
* value. For example, the scheduler uses
|
* value. `* const Something` or `std::optional<Something>` would be
|
||||||
* `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
|
* good choices for "optional" types.
|
||||||
* remember which goals correspond to which outputs. `* const Something`
|
|
||||||
* or `std::optional<Something>` would also be good choices for
|
|
||||||
* "optional" types.
|
|
||||||
*/
|
*/
|
||||||
template<typename V>
|
template<typename V>
|
||||||
struct DerivedPathMap {
|
struct DerivedPathMap {
|
||||||
|
|
|
@ -18,6 +18,4 @@ clearStore
|
||||||
|
|
||||||
drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
|
drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
|
||||||
|
|
||||||
out2=$(nix build "${drvDep}^out^out" --no-link)
|
expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
|
||||||
|
|
||||||
test $out1 == $out2
|
|
||||||
|
|
|
@ -6,6 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
|
|
||||||
out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
|
expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
|
||||||
|
|
||||||
diff -r $out1 $out2
|
# diff -r $out1 $out2
|
||||||
|
|
Loading…
Reference in a new issue