From eb6db4fd384f34ca426116cd353c02af7d0f9214 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Jul 2021 15:43:08 +0200 Subject: [PATCH] buildPaths(): Add an evalStore argument With this, we don't have to copy the entire .drv closure to the destination store ahead of time (or at all). Instead, buildPaths() reads .drv files from the eval store and copies inputSrcs to the destination store if it needs to build a derivation. Issue #5025. --- src/libcmd/installables.cc | 2 +- src/libstore/build/derivation-goal.cc | 26 ++++++++++++++------- src/libstore/build/entry-points.cc | 10 ++++---- src/libstore/build/local-derivation-goal.cc | 4 +++- src/libstore/build/worker.cc | 3 ++- src/libstore/build/worker.hh | 3 ++- src/libstore/legacy-ssh-store.cc | 5 +++- src/libstore/realisation.hh | 2 +- src/libstore/remote-store.cc | 5 +++- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 3 ++- src/nix-build/nix-build.cc | 10 +++----- src/nix/develop.cc | 18 +++++++------- 13 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 00f8105ae..e3ce564b0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -766,7 +766,7 @@ BuiltPaths build(ref evalStore, ref store, Realise mode, if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild, bMode); + store->buildPaths(pathsToBuild, bMode, evalStore); return getBuiltPaths(evalStore, store, pathsToBuild); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7df597400..dad8717a2 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -165,7 +165,7 @@ void DerivationGoal::getDerivation() /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { loadDerivation(); return; } @@ -188,12 +188,12 @@ void DerivationGoal::loadDerivation() /* `drvPath' should already be a root, but let's be on the safe side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); + worker.evalStore.addTempRoot(drvPath); - assert(worker.store.isValidPath(drvPath)); + assert(worker.evalStore.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::make_unique(worker.store.derivationFromPath(drvPath)); + drv = std::make_unique(worker.evalStore.derivationFromPath(drvPath)); haveDerivation(); } @@ -212,8 +212,8 @@ void DerivationGoal::haveDerivation() if (i.second.second) worker.store.addTempRoot(*i.second.second); - auto outputHashes = staticOutputHashes(worker.store, *drv); - for (auto &[outputName, outputHash] : outputHashes) + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + for (auto & [outputName, outputHash] : outputHashes) initialOutputs.insert({ outputName, InitialOutput{ @@ -337,6 +337,16 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + /* Copy the input sources from the eval store to the build + store. */ + if (&worker.evalStore != &worker.store) { + RealisedPath::Set inputSrcs, inputClosure; + for (auto & i : drv->inputSrcs) + inputSrcs.insert(i); + RealisedPath::closure(worker.evalStore, inputSrcs, inputClosure); + copyClosure(worker.evalStore, worker.store, inputClosure); + } + for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) @@ -478,8 +488,8 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + assert(worker.evalStore.isValidPath(drvPath)); + auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); for (auto & j : wantedDepOutputs) { if (outputs.count(j) > 0) { auto optRealizedInput = outputs.at(j); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 732d4785d..96deb81d1 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,9 +6,9 @@ namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) { - Worker worker(*this); + Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; for (auto & br : reqs) { @@ -51,7 +51,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this); + Worker worker(*this, *this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; @@ -93,7 +93,7 @@ void Store::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; @@ -111,7 +111,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 11c99d9f0..990ff60b7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1254,8 +1254,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo return next->queryRealisation(id); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override { + assert(!evalStore); + if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0f2ade348..a7a6b92a6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -9,11 +9,12 @@ namespace nix { -Worker::Worker(Store & store) +Worker::Worker(Store & store, Store & evalStore) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) , store(store) + , evalStore(evalStore) { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 918de35f6..6a3b99c02 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -110,6 +110,7 @@ public: bool checkMismatch; Store & store; + Store & evalStore; std::unique_ptr hook; @@ -131,7 +132,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(Store & store); + Worker(Store & store, Store & evalStore); ~Worker(); /* Make a goal (with caching). */ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index edaf75136..45eed5707 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,8 +267,11 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override { + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + auto conn(connections->get()); conn->to << cmdBuildPaths; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 05d2bc44f..9070a6ee2 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -45,7 +45,7 @@ struct Realisation { size_t checkSignatures(const PublicKeys & publicKeys) const; static std::set closure(Store &, const std::set &); - static void closure(Store &, const std::set &, std::set& res); + static void closure(Store &, const std::set &, std::set & res); bool isCompatibleWith(const Realisation & other) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cd2d718ec..77b34d000 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -705,8 +705,11 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) { + if (evalStore && evalStore.get() != this) + throw Error("building on a remote store is incompatible with '--eval-store'"); + auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 293393c32..fbec40b4b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 80edac244..3ad74f35c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -497,7 +497,8 @@ public: not derivations, substitute them. */ virtual void buildPaths( const std::vector & paths, - BuildMode buildMode = bmNormal); + BuildMode buildMode = bmNormal, + std::shared_ptr evalStore = nullptr); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3b791e619..848b108dd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -341,7 +341,7 @@ static void main_nix_build(int argc, char * * argv) printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (!dryRun) - store->buildPaths(paths, buildMode); + store->buildPaths(paths, buildMode, evalStore); }; if (runEnv) { @@ -397,8 +397,6 @@ static void main_nix_build(int argc, char * * argv) pathsToCopy.insert(src); } - copyClosure(*evalStore, *store, pathsToCopy); - buildPaths(pathsToBuild); if (dryRun) return; @@ -449,7 +447,7 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = store->queryPartialDerivationOutputMap(depDrvPath); + auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); for (auto & i : wantedDepOutputs) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); @@ -564,8 +562,6 @@ static void main_nix_build(int argc, char * * argv) drvMap[drvPath] = {drvMap.size(), {outputName}}; } - copyClosure(*evalStore, *store, drvsToCopy); - buildPaths(pathsToBuild); if (dryRun) return; @@ -578,7 +574,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index cb173b91b..9a93cdb03 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -171,15 +171,15 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) +static StorePath getDerivationEnvironment(ref store, ref evalStore, const StorePath & drvPath) { - auto drv = store->derivationFromPath(drvPath); + auto drv = evalStore->derivationFromPath(drvPath); auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); drv.args = {store->printStorePath(getEnvShPath)}; @@ -205,7 +205,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; drv.env[output.first] = ""; } - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); for (auto & output : drv.outputs) { auto outPath = store->makeOutputPath(output.first, h, drv.name); @@ -214,12 +214,12 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) } } - auto shellDrvPath = writeDerivation(*store, drv); + auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); - for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { + for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); @@ -347,7 +347,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, drvPath); + return getDerivationEnvironment(store, getEvalStore(), drvPath); } } @@ -361,7 +361,7 @@ struct Common : InstallableCommand, MixProfile debug("reading environment file '%s'", strPath); - return {BuildEnvironment::fromJSON(readFile(strPath)), strPath}; + return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath}; } };