forked from lix-project/lix
Only substitute wanted outputs of a derivation
If a derivation has multiple outputs, then we only want to download those outputs that are actuallty needed. So if we do "nix-build -A openssl.man", then only the "man" output should be downloaded. Likewise if another package depends on ${openssl.man}. The tricky part is that different derivations can depend on different outputs of a given derivation, so we may need to restart the corresponding derivation goal if that happens.
This commit is contained in:
parent
46a369ad95
commit
8d8d47abd2
|
@ -240,7 +240,7 @@ public:
|
||||||
~Worker();
|
~Worker();
|
||||||
|
|
||||||
/* Make a goal (with caching). */
|
/* Make a goal (with caching). */
|
||||||
GoalPtr makeDerivationGoal(const Path & drvPath, bool repair = false);
|
GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false);
|
||||||
GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
|
GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
|
||||||
|
|
||||||
/* Remove a dead goal. */
|
/* Remove a dead goal. */
|
||||||
|
@ -756,6 +756,13 @@ private:
|
||||||
/* The path of the derivation. */
|
/* The path of the derivation. */
|
||||||
Path drvPath;
|
Path drvPath;
|
||||||
|
|
||||||
|
/* The specific outputs that we need to build. Empty means all of
|
||||||
|
them. */
|
||||||
|
StringSet wantedOutputs;
|
||||||
|
|
||||||
|
/* Whether additional wanted outputs have been added. */
|
||||||
|
bool needRestart;
|
||||||
|
|
||||||
/* The derivation stored at drvPath. */
|
/* The derivation stored at drvPath. */
|
||||||
Derivation drv;
|
Derivation drv;
|
||||||
|
|
||||||
|
@ -831,7 +838,7 @@ private:
|
||||||
const static int childSetupFailed = 189;
|
const static int childSetupFailed = 189;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
DerivationGoal(const Path & drvPath, Worker & worker, bool repair = false);
|
DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
|
||||||
~DerivationGoal();
|
~DerivationGoal();
|
||||||
|
|
||||||
void cancel();
|
void cancel();
|
||||||
|
@ -843,6 +850,9 @@ public:
|
||||||
return drvPath;
|
return drvPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Add wanted outputs to an already existing derivation goal. */
|
||||||
|
void addWantedOutputs(const StringSet & outputs);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/* The states. */
|
/* The states. */
|
||||||
void init();
|
void init();
|
||||||
|
@ -897,8 +907,10 @@ private:
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker, bool repair)
|
DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair)
|
||||||
: Goal(worker)
|
: Goal(worker)
|
||||||
|
, wantedOutputs(wantedOutputs)
|
||||||
|
, needRestart(false)
|
||||||
, fLogFile(0)
|
, fLogFile(0)
|
||||||
, bzLogFile(0)
|
, bzLogFile(0)
|
||||||
, useChroot(false)
|
, useChroot(false)
|
||||||
|
@ -967,6 +979,23 @@ void DerivationGoal::work()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
|
||||||
|
{
|
||||||
|
/* If we already want all outputs, there is nothing to do. */
|
||||||
|
if (wantedOutputs.empty()) return;
|
||||||
|
|
||||||
|
if (outputs.empty()) {
|
||||||
|
wantedOutputs.clear();
|
||||||
|
needRestart = true;
|
||||||
|
} else
|
||||||
|
foreach (StringSet::const_iterator, i, outputs)
|
||||||
|
if (wantedOutputs.find(*i) == wantedOutputs.end()) {
|
||||||
|
wantedOutputs.insert(*i);
|
||||||
|
needRestart = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void DerivationGoal::init()
|
void DerivationGoal::init()
|
||||||
{
|
{
|
||||||
trace("init");
|
trace("init");
|
||||||
|
@ -1043,6 +1072,12 @@ void DerivationGoal::outputsSubstituted()
|
||||||
|
|
||||||
nrFailed = nrNoSubstituters = 0;
|
nrFailed = nrNoSubstituters = 0;
|
||||||
|
|
||||||
|
if (needRestart) {
|
||||||
|
needRestart = false;
|
||||||
|
haveDerivation();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (checkPathValidity(false, repair).size() == 0) {
|
if (checkPathValidity(false, repair).size() == 0) {
|
||||||
if (repair) repairClosure(); else amDone(ecSuccess);
|
if (repair) repairClosure(); else amDone(ecSuccess);
|
||||||
return;
|
return;
|
||||||
|
@ -1051,9 +1086,13 @@ void DerivationGoal::outputsSubstituted()
|
||||||
/* Otherwise, at least one of the output paths could not be
|
/* Otherwise, at least one of the output paths could not be
|
||||||
produced using a substitute. So we have to build instead. */
|
produced using a substitute. So we have to build instead. */
|
||||||
|
|
||||||
|
/* Make sure checkPathValidity() from now on checks all
|
||||||
|
outputs. */
|
||||||
|
wantedOutputs = PathSet();
|
||||||
|
|
||||||
/* The inputs must be built before we can build this goal. */
|
/* The inputs must be built before we can build this goal. */
|
||||||
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
|
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
|
||||||
addWaitee(worker.makeDerivationGoal(i->first, repair));
|
addWaitee(worker.makeDerivationGoal(i->first, i->second, repair));
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, drv.inputSrcs)
|
foreach (PathSet::iterator, i, drv.inputSrcs)
|
||||||
addWaitee(worker.makeSubstitutionGoal(*i));
|
addWaitee(worker.makeSubstitutionGoal(*i));
|
||||||
|
@ -1103,7 +1142,7 @@ void DerivationGoal::repairClosure()
|
||||||
if (drvPath2 == "")
|
if (drvPath2 == "")
|
||||||
addWaitee(worker.makeSubstitutionGoal(*i, true));
|
addWaitee(worker.makeSubstitutionGoal(*i, true));
|
||||||
else
|
else
|
||||||
addWaitee(worker.makeDerivationGoal(drvPath2, true));
|
addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (waitees.empty()) {
|
if (waitees.empty()) {
|
||||||
|
@ -2385,6 +2424,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
|
||||||
{
|
{
|
||||||
PathSet result;
|
PathSet result;
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs) {
|
foreach (DerivationOutputs::iterator, i, drv.outputs) {
|
||||||
|
if (!wantOutput(i->first, wantedOutputs)) continue;
|
||||||
bool good =
|
bool good =
|
||||||
worker.store.isValidPath(i->second.path) &&
|
worker.store.isValidPath(i->second.path) &&
|
||||||
(!checkHash || worker.store.pathContentsGood(i->second.path));
|
(!checkHash || worker.store.pathContentsGood(i->second.path));
|
||||||
|
@ -2851,14 +2891,15 @@ Worker::~Worker()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
GoalPtr Worker::makeDerivationGoal(const Path & path, bool repair)
|
GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair)
|
||||||
{
|
{
|
||||||
GoalPtr goal = derivationGoals[path].lock();
|
GoalPtr goal = derivationGoals[path].lock();
|
||||||
if (!goal) {
|
if (!goal) {
|
||||||
goal = GoalPtr(new DerivationGoal(path, *this, repair));
|
goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair));
|
||||||
derivationGoals[path] = goal;
|
derivationGoals[path] = goal;
|
||||||
wakeUp(goal);
|
wakeUp(goal);
|
||||||
}
|
} else
|
||||||
|
(dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
|
||||||
return goal;
|
return goal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3200,7 +3241,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, bool repair)
|
||||||
foreach (PathSet::const_iterator, i, drvPaths) {
|
foreach (PathSet::const_iterator, i, drvPaths) {
|
||||||
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
|
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
|
||||||
if (isDerivation(i2.first))
|
if (isDerivation(i2.first))
|
||||||
goals.insert(worker.makeDerivationGoal(i2.first, repair));
|
goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair));
|
||||||
else
|
else
|
||||||
goals.insert(worker.makeSubstitutionGoal(*i, repair));
|
goals.insert(worker.makeSubstitutionGoal(*i, repair));
|
||||||
}
|
}
|
||||||
|
|
|
@ -261,7 +261,7 @@ DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
|
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
|
||||||
{
|
{
|
||||||
return outputs.empty()
|
return outputs.empty()
|
||||||
? drvPath
|
? drvPath
|
||||||
|
@ -269,4 +269,10 @@ Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
bool wantOutput(const string & output, const std::set<string> & wanted)
|
||||||
|
{
|
||||||
|
return wanted.empty() || wanted.find(output) != wanted.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -85,7 +85,9 @@ extern DrvHashes drvHashes;
|
||||||
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
|
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
|
||||||
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
|
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
|
||||||
|
|
||||||
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs);
|
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
|
||||||
|
|
||||||
|
bool wantOutput(const string & output, const std::set<string> & wanted);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -93,12 +93,13 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
|
||||||
Derivation drv = derivationFromPath(store, i2.first);
|
Derivation drv = derivationFromPath(store, i2.first);
|
||||||
|
|
||||||
PathSet invalid;
|
PathSet invalid;
|
||||||
// FIXME: only fetch the desired outputs
|
|
||||||
foreach (DerivationOutputs::iterator, j, drv.outputs)
|
foreach (DerivationOutputs::iterator, j, drv.outputs)
|
||||||
if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
|
if (wantOutput(j->first, i2.second)
|
||||||
|
&& !store.isValidPath(j->second.path))
|
||||||
|
invalid.insert(j->second.path);
|
||||||
if (invalid.empty()) continue;
|
if (invalid.empty()) continue;
|
||||||
|
|
||||||
todoDrv.insert(i2.first);
|
todoDrv.insert(*i);
|
||||||
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
|
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -115,26 +116,32 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
|
||||||
store.querySubstitutablePathInfos(query, infos);
|
store.querySubstitutablePathInfos(query, infos);
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, todoDrv) {
|
foreach (PathSet::iterator, i, todoDrv) {
|
||||||
// FIXME: cache this
|
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
|
||||||
Derivation drv = derivationFromPath(store, *i);
|
|
||||||
|
|
||||||
|
// FIXME: cache this
|
||||||
|
Derivation drv = derivationFromPath(store, i2.first);
|
||||||
|
|
||||||
|
PathSet outputs;
|
||||||
bool mustBuild = false;
|
bool mustBuild = false;
|
||||||
if (settings.useSubstitutes) {
|
if (settings.useSubstitutes) {
|
||||||
foreach (DerivationOutputs::iterator, j, drv.outputs)
|
foreach (DerivationOutputs::iterator, j, drv.outputs) {
|
||||||
|
if (!wantOutput(j->first, i2.second)) continue;
|
||||||
if (!store.isValidPath(j->second.path) &&
|
if (!store.isValidPath(j->second.path) &&
|
||||||
infos.find(j->second.path) == infos.end())
|
infos.find(j->second.path) == infos.end())
|
||||||
mustBuild = true;
|
mustBuild = true;
|
||||||
|
else
|
||||||
|
outputs.insert(j->second.path);
|
||||||
|
}
|
||||||
} else
|
} else
|
||||||
mustBuild = true;
|
mustBuild = true;
|
||||||
|
|
||||||
if (mustBuild) {
|
if (mustBuild) {
|
||||||
willBuild.insert(*i);
|
willBuild.insert(i2.first);
|
||||||
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
||||||
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
|
foreach (DerivationInputs::iterator, j, drv.inputDrvs)
|
||||||
todo.insert(i->first);
|
todo.insert(makeDrvPathWithOutputs(j->first, j->second));
|
||||||
} else
|
} else
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
todoNonDrv.insert(outputs.begin(), outputs.end());
|
||||||
todoNonDrv.insert(i->second.path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, todoNonDrv) {
|
foreach (PathSet::iterator, i, todoNonDrv) {
|
||||||
|
|
Loading…
Reference in a new issue