nix-build: Copy drv closure between eval store and build store

This commit is contained in:
Eelco Dolstra 2021-07-16 09:37:33 +02:00
parent 2ff3035cf4
commit e9848beca7
5 changed files with 40 additions and 32 deletions

View file

@ -12,7 +12,6 @@
#include "eval-cache.hh"
#include "url.hh"
#include "registry.hh"
#include "remote-store.hh"
#include <regex>
#include <queue>
@ -393,31 +392,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
for (auto & i : drvsToOutputs)
res.push_back(DerivedPath::Built { i.first, i.second });
// FIXME: Temporary hack
if (state->store != state->buildStore) {
RealisedPath::Set closure;
RealisedPath::closure(*state->store, drvsToCopy, closure);
if (dynamic_cast<RemoteStore *>(&*state->buildStore)) {
StorePathSet closure2;
for (auto & p : closure)
closure2.insert(p.path());
auto valid = state->buildStore->queryValidPaths(closure2);
StorePathSet missing;
for (auto & p : closure2)
if (!valid.count(p)) missing.insert(p);
if (!missing.empty()) {
auto source = sinkToSource([&](Sink & sink) {
state->store->exportPaths(missing, sink);
});
state->buildStore->importPaths(*source, NoCheckSigs);
}
} else
copyPaths(state->store, state->buildStore, closure);
}
copyClosure(state->store, state->buildStore, drvsToCopy);
return res;
}

View file

@ -9,6 +9,7 @@
#include "url.hh"
#include "archive.hh"
#include "callback.hh"
#include "remote-store.hh"
#include <regex>
@ -881,6 +882,16 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
// FIXME: Temporary hack to copy closures in a single round-trip.
if (dynamic_cast<RemoteStore *>(&*dstStore)) {
if (!missing.empty()) {
auto source = sinkToSource([&](Sink & sink) {
srcStore->exportPaths(missing, sink);
});
dstStore->importPaths(*source, NoCheckSigs);
}
return pathsMap;
}
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@ -958,6 +969,22 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
return pathsMap;
}
void copyClosure(
ref<Store> srcStore,
ref<Store> dstStore,
const RealisedPath::Set & paths,
RepairFlag repair,
CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
if (srcStore == dstStore) return;
RealisedPath::Set closure;
RealisedPath::closure(*srcStore, paths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;

View file

@ -773,6 +773,12 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Copy the closure of `paths` from `srcStore` to `dstStore`. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const RealisedPath::Set & paths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered

View file

@ -532,6 +532,7 @@ static void main_nix_build(int argc, char * * argv)
std::vector<StorePathWithOutputs> pathsToBuild;
std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered;
RealisedPath::Set drvsToCopy;
std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
@ -544,14 +545,16 @@ static void main_nix_build(int argc, char * * argv)
pathsToBuild.push_back({drvPath, {outputName}});
pathsToBuildOrdered.push_back({drvPath, {outputName}});
drvsToCopy.insert(drvPath);
auto i = drvMap.find(drvPath);
if (i != drvMap.end())
i->second.second.insert(outputName);
else {
else
drvMap[drvPath] = {drvMap.size(), {outputName}};
}
}
copyClosure(state->store, state->buildStore, drvsToCopy);
buildPaths(pathsToBuild);

View file

@ -54,10 +54,7 @@ static int main_nix_copy_closure(int argc, char ** argv)
for (auto & path : storePaths)
storePaths2.insert(from->followLinksToStorePath(path));
RealisedPath::Set closure;
RealisedPath::closure(*from, storePaths2, closure);
copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
copyClosure(from, to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes);
return 0;
}