From 1aba0bf0fa831ffee628ae50730eade5b19a544f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jun 2012 16:58:15 -0400 Subject: [PATCH] nix-store -r: do substitutions in parallel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I.e. when multiple non-derivation arguments are passed to ‘nix-store -r’ to be substituted, do them in parallel. --- src/libexpr/primops.cc | 2 +- src/libstore/build.cc | 13 +++++++++---- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 8 +++++--- src/libstore/worker-protocol.hh | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-env/user-env.cc | 4 ++-- src/nix-store/nix-store.cc | 12 +++++------- src/nix-worker/nix-worker.cc | 4 ++-- 11 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4a171a2ae..7587dccea 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -53,7 +53,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v) try { /* !!! If using a substitute, we only need to fetch the selected output of this derivation. */ - store->buildDerivations(singleton(ctx)); + store->buildPaths(singleton(ctx)); } catch (Error & e) { throw ImportError(e.msg()); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 246e0d9da..d5bbd540b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2275,6 +2275,8 @@ public: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data); void handleEOF(int fd); + + Path getStorePath() { return storePath; } }; @@ -2938,7 +2940,7 @@ unsigned int Worker::exitStatus() ////////////////////////////////////////////////////////////////////// -void LocalStore::buildDerivations(const PathSet & drvPaths) +void LocalStore::buildPaths(const PathSet & drvPaths) { startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths)); @@ -2947,7 +2949,10 @@ void LocalStore::buildDerivations(const PathSet & drvPaths) Goals goals; foreach (PathSet::const_iterator, i, drvPaths) - goals.insert(worker.makeDerivationGoal(*i)); + if (isDerivation(*i)) + goals.insert(worker.makeDerivationGoal(*i)); + else + goals.insert(worker.makeSubstitutionGoal(*i)); worker.run(goals); @@ -2955,8 +2960,8 @@ void LocalStore::buildDerivations(const PathSet & drvPaths) foreach (Goals::iterator, i, goals) if ((*i)->getExitCode() == Goal::ecFailed) { DerivationGoal * i2 = dynamic_cast(i->get()); - assert(i2); - failed.insert(i2->getDrvPath()); + if (i2) failed.insert(i2->getDrvPath()); + else failed.insert(dynamic_cast(i->get())->getStorePath()); } if (!failed.empty()) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0ff3e7b39..aa8e8582f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -150,7 +150,7 @@ public: Paths importPaths(bool requireSignature, Source & source); - void buildDerivations(const PathSet & drvPaths); + void buildPaths(const PathSet & paths); void ensurePath(const Path & path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cbcf86054..5e5561a6a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -397,10 +397,10 @@ Paths RemoteStore::importPaths(bool requireSignature, Source & source) } -void RemoteStore::buildDerivations(const PathSet & drvPaths) +void RemoteStore::buildPaths(const PathSet & drvPaths) { openConnection(); - writeInt(wopBuildDerivations, to); + writeInt(wopBuildPaths, to); writeStrings(drvPaths, to); processStderr(); readInt(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 823e694dd..e9f40da6d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -60,7 +60,7 @@ public: Paths importPaths(bool requireSignature, Source & source); - void buildDerivations(const PathSet & drvPaths); + void buildPaths(const PathSet & paths); void ensurePath(const Path & path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index fa766d12e..bf3269f57 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -172,13 +172,15 @@ public: the Nix store. */ virtual Paths importPaths(bool requireSignature, Source & source) = 0; - /* Ensure that the output paths of the derivation are valid. If + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If they are already valid, this is a no-op. Otherwise, validity can be reached in two ways. First, if the output paths is substitutable, then build the path that way. Second, the output paths can be created by running the builder, after - recursively building any sub-derivations. */ - virtual void buildDerivations(const PathSet & drvPaths) = 0; + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + virtual void buildPaths(const PathSet & paths) = 0; /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c1ea7f758..6a5f0ed40 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -22,7 +22,7 @@ typedef enum { wopQueryReferrers = 6, wopAddToStore = 7, wopAddTextToStore = 8, - wopBuildDerivations = 9, + wopBuildPaths = 9, wopEnsurePath = 10, wopAddTempRoot = 11, wopAddIndirectRoot = 12, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a8d9076cf..7aa6276e3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -695,7 +695,7 @@ static void opSet(Globals & globals, PathSet paths = singleton(drv.queryDrvPath(globals.state)); printMissing(*store, paths); if (globals.dryRun) return; - store->buildDerivations(paths); + store->buildPaths(paths); } else { printMissing(*store, singleton(drv.queryOutPath(globals.state))); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 4480a17aa..510c5ca38 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -45,7 +45,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.insert(i->queryDrvPath(state)); debug(format("building user environment dependencies")); - store->buildDerivations(drvsToBuild); + store->buildPaths(drvsToBuild); /* Construct the whole top level derivation. */ PathSet references; @@ -132,7 +132,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Realise the resulting store expression. */ debug("building user environment"); - store->buildDerivations(singleton(topLevelDrv.queryDrvPath(state))); + store->buildPaths(singleton(topLevelDrv.queryDrvPath(state))); /* Switch the current user environment to the output path. */ PathLocks lock; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4768c2e7a..23863525f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -62,7 +62,7 @@ static Path useDeriver(Path path) static PathSet realisePath(const Path & path) { if (isDerivation(path)) { - store->buildDerivations(singleton(path)); + store->buildPaths(singleton(path)); Derivation drv = derivationFromPath(*store, path); PathSet outputs; @@ -101,13 +101,11 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; - /* Build all derivations at the same time to exploit parallelism. */ - PathSet drvPaths; - foreach (Strings::iterator, i, opArgs) - if (isDerivation(*i)) drvPaths.insert(*i); - store->buildDerivations(drvPaths); + /* Build all paths at the same time to exploit parallelism. */ + PathSet paths(opArgs.begin(), opArgs.end()); + store->buildPaths(paths); - foreach (Strings::iterator, i, opArgs) { + foreach (Paths::iterator, i, opArgs) { PathSet paths = realisePath(*i); foreach (PathSet::iterator, j, paths) cout << format("%1%\n") % *j; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 6131e73e6..2f0a2ab20 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -415,10 +415,10 @@ static void performOp(unsigned int clientVersion, break; } - case wopBuildDerivations: { + case wopBuildPaths: { PathSet drvs = readStorePaths(from); startWork(); - store->buildDerivations(drvs); + store->buildPaths(drvs); stopWork(); writeInt(1, to); break;