Allow substituting drv outputs when building

This commit is contained in:
regnat 2020-11-09 15:40:10 +01:00
parent 5d1c05b075
commit df9d4f88d5
5 changed files with 185 additions and 12 deletions

View file

@ -246,13 +246,18 @@ void DerivationGoal::haveDerivation()
through substitutes. If that doesn't work, we'll build through substitutes. If that doesn't work, we'll build
them. */ them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & [_, status] : initialOutputs) { for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue; if (!status.wanted) continue;
if (!status.known) { if (!status.known)
warn("do not know how to query for unknown floating content-addressed derivation output yet"); addWaitee(
/* Nothing to wait for; tail call */ upcast_goal(
return DerivationGoal::gaveUpOnSubstitution(); worker.makeDrvOutputSubstitutionGoal(
} DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
)
)
);
else
addWaitee(upcast_goal(worker.makePathSubstitutionGoal( addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path, status.known->path,
buildMode == bmRepair ? Repair : NoRepair, buildMode == bmRepair ? Repair : NoRepair,

View file

@ -0,0 +1,95 @@
#include "drv-output-substitution-goal.hh"
#include "worker.hh"
#include "substitution-goal.hh"
namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, id(id)
{
state = &DrvOutputSubstitutionGoal::init;
name = fmt("substitution of '%s'", id.to_string());
trace("created");
}
void DrvOutputSubstitutionGoal::init()
{
trace("init");
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
void DrvOutputSubstitutionGoal::tryNext()
{
trace("Trying next substituter");
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
worker.updateProgress();
}
return;
}
auto sub = subs.front();
subs.pop_front();
// FIXME: Make async
outputInfo = sub->queryRealisation(id);
if (!outputInfo) {
tryNext();
return;
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
if (waitees.empty()) outPathValid();
else state = &DrvOutputSubstitutionGoal::outPathValid;
}
void DrvOutputSubstitutionGoal::outPathValid()
{
assert(outputInfo);
trace("Output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
return;
}
worker.store.registerDrvOutput(*outputInfo);
finished();
}
void DrvOutputSubstitutionGoal::finished()
{
trace("finished");
amDone(ecSuccess);
}
string DrvOutputSubstitutionGoal::key()
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(id.to_string());
}
void DrvOutputSubstitutionGoal::work()
{
(this->*state)();
}
}

View file

@ -0,0 +1,50 @@
#pragma once
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
namespace nix {
class Worker;
// Substitution of a derivation output.
// This is done in three steps:
// 1. Fetch the output info from a substituter
// 2. Substitute the corresponding output path
// 3. Register the output info
class DrvOutputSubstitutionGoal : public Goal {
private:
// The drv output we're trying to substitue
DrvOutput id;
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
std::optional<Realisation> outputInfo;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* Whether a substituter failed. */
bool substituterFailed = false;
public:
DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
typedef void (DrvOutputSubstitutionGoal::*GoalState)();
GoalState state;
void init();
void tryNext();
void outPathValid();
void finished();
void timedOut(Error && ex) override { abort(); };
string key() override;
void work() override;
};
}

View file

@ -1,6 +1,7 @@
#include "machines.hh" #include "machines.hh"
#include "worker.hh" #include "worker.hh"
#include "substitution-goal.hh" #include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh" #include "local-derivation-goal.hh"
#include "hook-instance.hh" #include "hook-instance.hh"
@ -90,8 +91,20 @@ std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const Sto
return goal; return goal;
} }
template<typename G> std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap) {
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{ {
/* !!! inefficient */ /* !!! inefficient */
for (auto i = goalMap.begin(); for (auto i = goalMap.begin();
@ -111,6 +124,8 @@ void Worker::removeGoal(GoalPtr goal)
nix::removeGoal(drvGoal, derivationGoals); nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals); nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else else
assert(false); assert(false);
if (topGoals.find(goal) != topGoals.end()) { if (topGoals.find(goal) != topGoals.end()) {
@ -474,5 +489,8 @@ void Worker::markContentsGood(const StorePath & path)
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) { GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
return subGoal; return subGoal;
} }
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
return subGoal;
}
} }

View file

@ -4,6 +4,7 @@
#include "lock.hh" #include "lock.hh"
#include "store-api.hh" #include "store-api.hh"
#include "goal.hh" #include "goal.hh"
#include "realisation.hh"
#include <future> #include <future>
#include <thread> #include <thread>
@ -13,6 +14,7 @@ namespace nix {
/* Forward definition. */ /* Forward definition. */
struct DerivationGoal; struct DerivationGoal;
struct PathSubstitutionGoal; struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like /* Workaround for not being able to declare a something like
@ -24,6 +26,7 @@ struct PathSubstitutionGoal;
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */ is opaque. */
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
@ -73,6 +76,7 @@ private:
same derivation / path. */ same derivation / path. */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */ /* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal; WeakGoals waitingForAnyGoal;
@ -147,6 +151,7 @@ public:
/* substitution goal */ /* substitution goal */
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */ /* Remove a dead goal. */
void removeGoal(GoalPtr goal); void removeGoal(GoalPtr goal);