nix-store -r: do substitutions in parallel
I.e. when multiple non-derivation arguments are passed to ‘nix-store -r’ to be substituted, do them in parallel.
This commit is contained in:
parent
42f5a2fc29
commit
1aba0bf0fa
|
@ -53,7 +53,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
|
||||||
try {
|
try {
|
||||||
/* !!! If using a substitute, we only need to fetch
|
/* !!! If using a substitute, we only need to fetch
|
||||||
the selected output of this derivation. */
|
the selected output of this derivation. */
|
||||||
store->buildDerivations(singleton<PathSet>(ctx));
|
store->buildPaths(singleton<PathSet>(ctx));
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
throw ImportError(e.msg());
|
throw ImportError(e.msg());
|
||||||
}
|
}
|
||||||
|
|
|
@ -2275,6 +2275,8 @@ public:
|
||||||
/* Callback used by the worker to write to the log. */
|
/* Callback used by the worker to write to the log. */
|
||||||
void handleChildOutput(int fd, const string & data);
|
void handleChildOutput(int fd, const string & data);
|
||||||
void handleEOF(int fd);
|
void handleEOF(int fd);
|
||||||
|
|
||||||
|
Path getStorePath() { return storePath; }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -2938,7 +2940,7 @@ unsigned int Worker::exitStatus()
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::buildDerivations(const PathSet & drvPaths)
|
void LocalStore::buildPaths(const PathSet & drvPaths)
|
||||||
{
|
{
|
||||||
startNest(nest, lvlDebug,
|
startNest(nest, lvlDebug,
|
||||||
format("building %1%") % showPaths(drvPaths));
|
format("building %1%") % showPaths(drvPaths));
|
||||||
|
@ -2947,7 +2949,10 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
|
||||||
|
|
||||||
Goals goals;
|
Goals goals;
|
||||||
foreach (PathSet::const_iterator, i, drvPaths)
|
foreach (PathSet::const_iterator, i, drvPaths)
|
||||||
goals.insert(worker.makeDerivationGoal(*i));
|
if (isDerivation(*i))
|
||||||
|
goals.insert(worker.makeDerivationGoal(*i));
|
||||||
|
else
|
||||||
|
goals.insert(worker.makeSubstitutionGoal(*i));
|
||||||
|
|
||||||
worker.run(goals);
|
worker.run(goals);
|
||||||
|
|
||||||
|
@ -2955,8 +2960,8 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
|
||||||
foreach (Goals::iterator, i, goals)
|
foreach (Goals::iterator, i, goals)
|
||||||
if ((*i)->getExitCode() == Goal::ecFailed) {
|
if ((*i)->getExitCode() == Goal::ecFailed) {
|
||||||
DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get());
|
DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get());
|
||||||
assert(i2);
|
if (i2) failed.insert(i2->getDrvPath());
|
||||||
failed.insert(i2->getDrvPath());
|
else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!failed.empty())
|
if (!failed.empty())
|
||||||
|
|
|
@ -150,7 +150,7 @@ public:
|
||||||
|
|
||||||
Paths importPaths(bool requireSignature, Source & source);
|
Paths importPaths(bool requireSignature, Source & source);
|
||||||
|
|
||||||
void buildDerivations(const PathSet & drvPaths);
|
void buildPaths(const PathSet & paths);
|
||||||
|
|
||||||
void ensurePath(const Path & path);
|
void ensurePath(const Path & path);
|
||||||
|
|
||||||
|
|
|
@ -397,10 +397,10 @@ Paths RemoteStore::importPaths(bool requireSignature, Source & source)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void RemoteStore::buildDerivations(const PathSet & drvPaths)
|
void RemoteStore::buildPaths(const PathSet & drvPaths)
|
||||||
{
|
{
|
||||||
openConnection();
|
openConnection();
|
||||||
writeInt(wopBuildDerivations, to);
|
writeInt(wopBuildPaths, to);
|
||||||
writeStrings(drvPaths, to);
|
writeStrings(drvPaths, to);
|
||||||
processStderr();
|
processStderr();
|
||||||
readInt(from);
|
readInt(from);
|
||||||
|
|
|
@ -60,7 +60,7 @@ public:
|
||||||
|
|
||||||
Paths importPaths(bool requireSignature, Source & source);
|
Paths importPaths(bool requireSignature, Source & source);
|
||||||
|
|
||||||
void buildDerivations(const PathSet & drvPaths);
|
void buildPaths(const PathSet & paths);
|
||||||
|
|
||||||
void ensurePath(const Path & path);
|
void ensurePath(const Path & path);
|
||||||
|
|
||||||
|
|
|
@ -172,13 +172,15 @@ public:
|
||||||
the Nix store. */
|
the Nix store. */
|
||||||
virtual Paths importPaths(bool requireSignature, Source & source) = 0;
|
virtual Paths importPaths(bool requireSignature, Source & source) = 0;
|
||||||
|
|
||||||
/* Ensure that the output paths of the derivation are valid. If
|
/* For each path, if it's a derivation, build it. Building a
|
||||||
|
derivation means ensuring that the output paths are valid. If
|
||||||
they are already valid, this is a no-op. Otherwise, validity
|
they are already valid, this is a no-op. Otherwise, validity
|
||||||
can be reached in two ways. First, if the output paths is
|
can be reached in two ways. First, if the output paths is
|
||||||
substitutable, then build the path that way. Second, the
|
substitutable, then build the path that way. Second, the
|
||||||
output paths can be created by running the builder, after
|
output paths can be created by running the builder, after
|
||||||
recursively building any sub-derivations. */
|
recursively building any sub-derivations. For inputs that are
|
||||||
virtual void buildDerivations(const PathSet & drvPaths) = 0;
|
not derivations, substitute them. */
|
||||||
|
virtual void buildPaths(const PathSet & paths) = 0;
|
||||||
|
|
||||||
/* Ensure that a path is valid. If it is not currently valid, it
|
/* Ensure that a path is valid. If it is not currently valid, it
|
||||||
may be made valid by running a substitute (if defined for the
|
may be made valid by running a substitute (if defined for the
|
||||||
|
|
|
@ -22,7 +22,7 @@ typedef enum {
|
||||||
wopQueryReferrers = 6,
|
wopQueryReferrers = 6,
|
||||||
wopAddToStore = 7,
|
wopAddToStore = 7,
|
||||||
wopAddTextToStore = 8,
|
wopAddTextToStore = 8,
|
||||||
wopBuildDerivations = 9,
|
wopBuildPaths = 9,
|
||||||
wopEnsurePath = 10,
|
wopEnsurePath = 10,
|
||||||
wopAddTempRoot = 11,
|
wopAddTempRoot = 11,
|
||||||
wopAddIndirectRoot = 12,
|
wopAddIndirectRoot = 12,
|
||||||
|
|
|
@ -695,7 +695,7 @@ static void opSet(Globals & globals,
|
||||||
PathSet paths = singleton<PathSet>(drv.queryDrvPath(globals.state));
|
PathSet paths = singleton<PathSet>(drv.queryDrvPath(globals.state));
|
||||||
printMissing(*store, paths);
|
printMissing(*store, paths);
|
||||||
if (globals.dryRun) return;
|
if (globals.dryRun) return;
|
||||||
store->buildDerivations(paths);
|
store->buildPaths(paths);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
printMissing(*store, singleton<PathSet>(drv.queryOutPath(globals.state)));
|
printMissing(*store, singleton<PathSet>(drv.queryOutPath(globals.state)));
|
||||||
|
|
|
@ -45,7 +45,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
|
||||||
drvsToBuild.insert(i->queryDrvPath(state));
|
drvsToBuild.insert(i->queryDrvPath(state));
|
||||||
|
|
||||||
debug(format("building user environment dependencies"));
|
debug(format("building user environment dependencies"));
|
||||||
store->buildDerivations(drvsToBuild);
|
store->buildPaths(drvsToBuild);
|
||||||
|
|
||||||
/* Construct the whole top level derivation. */
|
/* Construct the whole top level derivation. */
|
||||||
PathSet references;
|
PathSet references;
|
||||||
|
@ -132,7 +132,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
|
||||||
|
|
||||||
/* Realise the resulting store expression. */
|
/* Realise the resulting store expression. */
|
||||||
debug("building user environment");
|
debug("building user environment");
|
||||||
store->buildDerivations(singleton<PathSet>(topLevelDrv.queryDrvPath(state)));
|
store->buildPaths(singleton<PathSet>(topLevelDrv.queryDrvPath(state)));
|
||||||
|
|
||||||
/* Switch the current user environment to the output path. */
|
/* Switch the current user environment to the output path. */
|
||||||
PathLocks lock;
|
PathLocks lock;
|
||||||
|
|
|
@ -62,7 +62,7 @@ static Path useDeriver(Path path)
|
||||||
static PathSet realisePath(const Path & path)
|
static PathSet realisePath(const Path & path)
|
||||||
{
|
{
|
||||||
if (isDerivation(path)) {
|
if (isDerivation(path)) {
|
||||||
store->buildDerivations(singleton<PathSet>(path));
|
store->buildPaths(singleton<PathSet>(path));
|
||||||
Derivation drv = derivationFromPath(*store, path);
|
Derivation drv = derivationFromPath(*store, path);
|
||||||
|
|
||||||
PathSet outputs;
|
PathSet outputs;
|
||||||
|
@ -101,13 +101,11 @@ static void opRealise(Strings opFlags, Strings opArgs)
|
||||||
|
|
||||||
if (dryRun) return;
|
if (dryRun) return;
|
||||||
|
|
||||||
/* Build all derivations at the same time to exploit parallelism. */
|
/* Build all paths at the same time to exploit parallelism. */
|
||||||
PathSet drvPaths;
|
PathSet paths(opArgs.begin(), opArgs.end());
|
||||||
foreach (Strings::iterator, i, opArgs)
|
store->buildPaths(paths);
|
||||||
if (isDerivation(*i)) drvPaths.insert(*i);
|
|
||||||
store->buildDerivations(drvPaths);
|
|
||||||
|
|
||||||
foreach (Strings::iterator, i, opArgs) {
|
foreach (Paths::iterator, i, opArgs) {
|
||||||
PathSet paths = realisePath(*i);
|
PathSet paths = realisePath(*i);
|
||||||
foreach (PathSet::iterator, j, paths)
|
foreach (PathSet::iterator, j, paths)
|
||||||
cout << format("%1%\n") % *j;
|
cout << format("%1%\n") % *j;
|
||||||
|
|
|
@ -415,10 +415,10 @@ static void performOp(unsigned int clientVersion,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case wopBuildDerivations: {
|
case wopBuildPaths: {
|
||||||
PathSet drvs = readStorePaths<PathSet>(from);
|
PathSet drvs = readStorePaths<PathSet>(from);
|
||||||
startWork();
|
startWork();
|
||||||
store->buildDerivations(drvs);
|
store->buildPaths(drvs);
|
||||||
stopWork();
|
stopWork();
|
||||||
writeInt(1, to);
|
writeInt(1, to);
|
||||||
break;
|
break;
|
||||||
|
|
Loading…
Reference in a new issue