Only substitute wanted outputs of a derivation

If a derivation has multiple outputs, then we only want to download
those outputs that are actuallty needed.  So if we do "nix-build -A
openssl.man", then only the "man" output should be downloaded.
Likewise if another package depends on ${openssl.man}.

The tricky part is that different derivations can depend on different
outputs of a given derivation, so we may need to restart the
corresponding derivation goal if that happens.
This commit is contained in:
Eelco Dolstra 2012-11-26 17:15:09 +01:00
parent 46a369ad95
commit 8d8d47abd2
4 changed files with 78 additions and 22 deletions

View file

@ -240,7 +240,7 @@ public:
~Worker(); ~Worker();
/* Make a goal (with caching). */ /* Make a goal (with caching). */
GoalPtr makeDerivationGoal(const Path & drvPath, bool repair = false); GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false);
GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
/* Remove a dead goal. */ /* Remove a dead goal. */
@ -756,6 +756,13 @@ private:
/* The path of the derivation. */ /* The path of the derivation. */
Path drvPath; Path drvPath;
/* The specific outputs that we need to build. Empty means all of
them. */
StringSet wantedOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart;
/* The derivation stored at drvPath. */ /* The derivation stored at drvPath. */
Derivation drv; Derivation drv;
@ -831,7 +838,7 @@ private:
const static int childSetupFailed = 189; const static int childSetupFailed = 189;
public: public:
DerivationGoal(const Path & drvPath, Worker & worker, bool repair = false); DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
~DerivationGoal(); ~DerivationGoal();
void cancel(); void cancel();
@ -843,6 +850,9 @@ public:
return drvPath; return drvPath;
} }
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
private: private:
/* The states. */ /* The states. */
void init(); void init();
@ -897,8 +907,10 @@ private:
}; };
DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker, bool repair) DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair)
: Goal(worker) : Goal(worker)
, wantedOutputs(wantedOutputs)
, needRestart(false)
, fLogFile(0) , fLogFile(0)
, bzLogFile(0) , bzLogFile(0)
, useChroot(false) , useChroot(false)
@ -967,6 +979,23 @@ void DerivationGoal::work()
} }
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
{
/* If we already want all outputs, there is nothing to do. */
if (wantedOutputs.empty()) return;
if (outputs.empty()) {
wantedOutputs.clear();
needRestart = true;
} else
foreach (StringSet::const_iterator, i, outputs)
if (wantedOutputs.find(*i) == wantedOutputs.end()) {
wantedOutputs.insert(*i);
needRestart = true;
}
}
void DerivationGoal::init() void DerivationGoal::init()
{ {
trace("init"); trace("init");
@ -1043,6 +1072,12 @@ void DerivationGoal::outputsSubstituted()
nrFailed = nrNoSubstituters = 0; nrFailed = nrNoSubstituters = 0;
if (needRestart) {
needRestart = false;
haveDerivation();
return;
}
if (checkPathValidity(false, repair).size() == 0) { if (checkPathValidity(false, repair).size() == 0) {
if (repair) repairClosure(); else amDone(ecSuccess); if (repair) repairClosure(); else amDone(ecSuccess);
return; return;
@ -1051,9 +1086,13 @@ void DerivationGoal::outputsSubstituted()
/* Otherwise, at least one of the output paths could not be /* Otherwise, at least one of the output paths could not be
produced using a substitute. So we have to build instead. */ produced using a substitute. So we have to build instead. */
/* Make sure checkPathValidity() from now on checks all
outputs. */
wantedOutputs = PathSet();
/* The inputs must be built before we can build this goal. */ /* The inputs must be built before we can build this goal. */
foreach (DerivationInputs::iterator, i, drv.inputDrvs) foreach (DerivationInputs::iterator, i, drv.inputDrvs)
addWaitee(worker.makeDerivationGoal(i->first, repair)); addWaitee(worker.makeDerivationGoal(i->first, i->second, repair));
foreach (PathSet::iterator, i, drv.inputSrcs) foreach (PathSet::iterator, i, drv.inputSrcs)
addWaitee(worker.makeSubstitutionGoal(*i)); addWaitee(worker.makeSubstitutionGoal(*i));
@ -1103,7 +1142,7 @@ void DerivationGoal::repairClosure()
if (drvPath2 == "") if (drvPath2 == "")
addWaitee(worker.makeSubstitutionGoal(*i, true)); addWaitee(worker.makeSubstitutionGoal(*i, true));
else else
addWaitee(worker.makeDerivationGoal(drvPath2, true)); addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true));
} }
if (waitees.empty()) { if (waitees.empty()) {
@ -2385,6 +2424,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
{ {
PathSet result; PathSet result;
foreach (DerivationOutputs::iterator, i, drv.outputs) { foreach (DerivationOutputs::iterator, i, drv.outputs) {
if (!wantOutput(i->first, wantedOutputs)) continue;
bool good = bool good =
worker.store.isValidPath(i->second.path) && worker.store.isValidPath(i->second.path) &&
(!checkHash || worker.store.pathContentsGood(i->second.path)); (!checkHash || worker.store.pathContentsGood(i->second.path));
@ -2851,14 +2891,15 @@ Worker::~Worker()
} }
GoalPtr Worker::makeDerivationGoal(const Path & path, bool repair) GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair)
{ {
GoalPtr goal = derivationGoals[path].lock(); GoalPtr goal = derivationGoals[path].lock();
if (!goal) { if (!goal) {
goal = GoalPtr(new DerivationGoal(path, *this, repair)); goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair));
derivationGoals[path] = goal; derivationGoals[path] = goal;
wakeUp(goal); wakeUp(goal);
} } else
(dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
return goal; return goal;
} }
@ -3200,7 +3241,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, bool repair)
foreach (PathSet::const_iterator, i, drvPaths) { foreach (PathSet::const_iterator, i, drvPaths) {
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
if (isDerivation(i2.first)) if (isDerivation(i2.first))
goals.insert(worker.makeDerivationGoal(i2.first, repair)); goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair));
else else
goals.insert(worker.makeSubstitutionGoal(*i, repair)); goals.insert(worker.makeSubstitutionGoal(*i, repair));
} }

View file

@ -261,7 +261,7 @@ DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
} }
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs) Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
{ {
return outputs.empty() return outputs.empty()
? drvPath ? drvPath
@ -269,4 +269,10 @@ Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
} }
bool wantOutput(const string & output, const std::set<string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();
}
} }

View file

@ -85,7 +85,9 @@ extern DrvHashes drvHashes;
typedef std::pair<string, std::set<string> > DrvPathWithOutputs; typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs); Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
bool wantOutput(const string & output, const std::set<string> & wanted);
} }

View file

@ -93,12 +93,13 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
Derivation drv = derivationFromPath(store, i2.first); Derivation drv = derivationFromPath(store, i2.first);
PathSet invalid; PathSet invalid;
// FIXME: only fetch the desired outputs
foreach (DerivationOutputs::iterator, j, drv.outputs) foreach (DerivationOutputs::iterator, j, drv.outputs)
if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path); if (wantOutput(j->first, i2.second)
&& !store.isValidPath(j->second.path))
invalid.insert(j->second.path);
if (invalid.empty()) continue; if (invalid.empty()) continue;
todoDrv.insert(i2.first); todoDrv.insert(*i);
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end()); if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
} }
@ -115,26 +116,32 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
store.querySubstitutablePathInfos(query, infos); store.querySubstitutablePathInfos(query, infos);
foreach (PathSet::iterator, i, todoDrv) { foreach (PathSet::iterator, i, todoDrv) {
// FIXME: cache this DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
Derivation drv = derivationFromPath(store, *i);
// FIXME: cache this
Derivation drv = derivationFromPath(store, i2.first);
PathSet outputs;
bool mustBuild = false; bool mustBuild = false;
if (settings.useSubstitutes) { if (settings.useSubstitutes) {
foreach (DerivationOutputs::iterator, j, drv.outputs) foreach (DerivationOutputs::iterator, j, drv.outputs) {
if (!wantOutput(j->first, i2.second)) continue;
if (!store.isValidPath(j->second.path) && if (!store.isValidPath(j->second.path) &&
infos.find(j->second.path) == infos.end()) infos.find(j->second.path) == infos.end())
mustBuild = true; mustBuild = true;
else
outputs.insert(j->second.path);
}
} else } else
mustBuild = true; mustBuild = true;
if (mustBuild) { if (mustBuild) {
willBuild.insert(*i); willBuild.insert(i2.first);
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
foreach (DerivationInputs::iterator, i, drv.inputDrvs) foreach (DerivationInputs::iterator, j, drv.inputDrvs)
todo.insert(i->first); todo.insert(makeDrvPathWithOutputs(j->first, j->second));
} else } else
foreach (DerivationOutputs::iterator, i, drv.outputs) todoNonDrv.insert(outputs.begin(), outputs.end());
todoNonDrv.insert(i->second.path);
} }
foreach (PathSet::iterator, i, todoNonDrv) { foreach (PathSet::iterator, i, todoNonDrv) {