lix-project/lix
20
55
Fork 31
Code Issues 344 Code Review (Gerrit) Projects 3 Releases Wiki Activity

buildPaths(): Add an evalStore argument

Browse source 
With this, we don't have to copy the entire .drv closure to the
destination store ahead of time (or at all). Instead, buildPaths()
reads .drv files from the eval store and copies inputSrcs to the
destination store if it needs to build a derivation.

Issue #5025.
This commit is contained in:
Eelco Dolstra 2021-07-19 15:43:08 +02:00
parent 668abd3e57
commit eb6db4fd38
13 changed files with 55 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/libcmd/installables.cc
View file

@ -766,7 +766,7 @@ BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
if (mode == Realise::Nothing)
printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs)
store->buildPaths(pathsToBuild, bMode);
store->buildPaths(pathsToBuild, bMode, evalStore);
return getBuiltPaths(evalStore, store, pathsToBuild);
}

26
src/libstore/build/derivation-goal.cc
View file

@ -165,7 +165,7 @@ void DerivationGoal::getDerivation()
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
@ -188,12 +188,12 @@ void DerivationGoal::loadDerivation()
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.store.addTempRoot(drvPath);
worker.evalStore.addTempRoot(drvPath);
assert(worker.store.isValidPath(drvPath));
assert(worker.evalStore.isValidPath(drvPath));
/* Get the derivation. */
drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
haveDerivation();
}
@ -212,8 +212,8 @@ void DerivationGoal::haveDerivation()
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
auto outputHashes = staticOutputHashes(worker.store, *drv);
for (auto &[outputName, outputHash] : outputHashes)
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({
outputName,
InitialOutput{
@ -337,6 +337,16 @@ void DerivationGoal::gaveUpOnSubstitution()
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
/* Copy the input sources from the eval store to the build
store. */
if (&worker.evalStore != &worker.store) {
RealisedPath::Set inputSrcs, inputClosure;
for (auto & i : drv->inputSrcs)
inputSrcs.insert(i);
RealisedPath::closure(worker.evalStore, inputSrcs, inputClosure);
copyClosure(worker.evalStore, worker.store, inputClosure);
}
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i)) continue;
if (!settings.useSubstitutes)
@ -478,8 +488,8 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.store.isValidPath(drvPath));
auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);

10
src/libstore/build/entry-points.cc
View file

@ -6,9 +6,9 @@
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
Worker worker(*this);
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (auto & br : reqs) {
@ -51,7 +51,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
@ -93,7 +93,7 @@ void Store::ensurePath(const StorePath & path)
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
@ -111,7 +111,7 @@ void Store::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};

4
src/libstore/build/local-derivation-goal.cc
View file

@ -1254,8 +1254,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
return next->queryRealisation(id);
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
assert(!evalStore);
if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths;

3
src/libstore/build/worker.cc
View file

@ -9,11 +9,12 @@
namespace nix {
Worker::Worker(Store & store)
Worker::Worker(Store & store, Store & evalStore)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;

3
src/libstore/build/worker.hh
View file

@ -110,6 +110,7 @@ public:
bool checkMismatch;
Store & store;
Store & evalStore;
std::unique_ptr<HookInstance> hook;
@ -131,7 +132,7 @@ public:
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(Store & store);
Worker(Store & store, Store & evalStore);
~Worker();
/* Make a goal (with caching). */

5
src/libstore/legacy-ssh-store.cc
View file

@ -267,8 +267,11 @@ public:
return status;
}
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) override
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
if (evalStore && evalStore.get() != this)
throw Error("building on an SSH store is incompatible with '--eval-store'");
auto conn(connections->get());
conn->to << cmdBuildPaths;

2
src/libstore/realisation.hh
View file

@ -45,7 +45,7 @@ struct Realisation {
size_t checkSignatures(const PublicKeys & publicKeys) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
bool isCompatibleWith(const Realisation & other) const;

5
src/libstore/remote-store.cc
View file

@ -705,8 +705,11 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons
}
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode)
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
if (evalStore && evalStore.get() != this)
throw Error("building on a remote store is incompatible with '--eval-store'");
auto conn(getConnection());
conn->to << wopBuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);

2
src/libstore/remote-store.hh
View file

@ -85,7 +85,7 @@ public:
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;

3
src/libstore/store-api.hh
View file

@ -497,7 +497,8 @@ public:
not derivations, substitute them. */
virtual void buildPaths(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal);
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file).

10
src/nix-build/nix-build.cc
View file

@ -341,7 +341,7 @@ static void main_nix_build(int argc, char * * argv)
printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!dryRun)
store->buildPaths(paths, buildMode);
store->buildPaths(paths, buildMode, evalStore);
};
if (runEnv) {
@ -397,8 +397,6 @@ static void main_nix_build(int argc, char * * argv)
pathsToCopy.insert(src);
}
copyClosure(*evalStore, *store, pathsToCopy);
buildPaths(pathsToBuild);
if (dryRun) return;
@ -449,7 +447,7 @@ static void main_nix_build(int argc, char * * argv)
if (env.count("__json")) {
StorePathSet inputs;
for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) {
auto outputs = store->queryPartialDerivationOutputMap(depDrvPath);
auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath);
for (auto & i : wantedDepOutputs) {
auto o = outputs.at(i);
store->computeFSClosure(*o, inputs);
@ -564,8 +562,6 @@ static void main_nix_build(int argc, char * * argv)
drvMap[drvPath] = {drvMap.size(), {outputName}};
}
copyClosure(*evalStore, *store, drvsToCopy);
buildPaths(pathsToBuild);
if (dryRun) return;
@ -578,7 +574,7 @@ static void main_nix_build(int argc, char * * argv)
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath);
auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath);
auto maybeOutputPath = builtOutputs.at(outputName);
assert(maybeOutputPath);

18
src/nix/develop.cc
View file

@ -171,15 +171,15 @@ const static std::string getEnvSh =
modified derivation with the same dependencies and nearly the same
initial environment variables, that just writes the resulting
environment to a file and exits. */
StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore, const StorePath & drvPath)
{
auto drv = store->derivationFromPath(drvPath);
auto drv = evalStore->derivationFromPath(drvPath);
auto builder = baseNameOf(drv.builder);
if (builder != "bash")
throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {});
drv.args = {store->printStorePath(getEnvShPath)};
@ -205,7 +205,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
drv.env[output.first] = "";
}
Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true));
for (auto & output : drv.outputs) {
auto outPath = store->makeOutputPath(output.first, h, drv.name);
@ -214,12 +214,12 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
}
}
auto shellDrvPath = writeDerivation(*store, drv);
auto shellDrvPath = writeDerivation(*evalStore, drv);
/* Build the derivation. */
store->buildPaths({DerivedPath::Built{shellDrvPath}});
store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore);
for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
assert(optPath);
auto & outPath = *optPath;
assert(store->isValidPath(outPath));
@ -347,7 +347,7 @@ struct Common : InstallableCommand, MixProfile
auto & drvPath = *drvs.begin();
return getDerivationEnvironment(store, drvPath);
return getDerivationEnvironment(store, getEvalStore(), drvPath);
}
}
@ -361,7 +361,7 @@ struct Common : InstallableCommand, MixProfile
debug("reading environment file '%s'", strPath);
return {BuildEnvironment::fromJSON(readFile(strPath)), strPath};
return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath};
}
};