lunaphied/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

copyPaths: Pass store by reference

Browse source
This commit is contained in:
Eelco Dolstra 2021-07-19 12:01:06 +02:00
parent 8d9f7048cd
commit 668abd3e57
8 changed files with 81 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/build-remote/build-remote.cc
View file

@ -270,7 +270,7 @@ connected:
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
} }
uploadLock = -1; uploadLock = -1;
@ -321,7 +321,7 @@ connected:
if (auto localStore = store.dynamic_pointer_cast<LocalStore>()) if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths) for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
} }
// XXX: Should be done as part of `copyPaths` // XXX: Should be done as part of `copyPaths`
for (auto & realisation : missingRealisations) { for (auto & realisation : missingRealisations) {

2
src/libstore/build/substitution-goal.cc
View file

@ -204,7 +204,7 @@ void PathSubstitutionGoal::tryToRun()
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id); PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), copyStorePath(*sub, worker.store,
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value(); promise.set_value();

100
src/libstore/store-api.cc
View file

@ -771,30 +771,34 @@ const Store::Stats & Store::getStats()
} }
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, void copyStorePath(
const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) Store & srcStore,
Store & dstStore,
const StorePath & storePath,
RepairFlag repair,
CheckSigsFlag checkSigs)
{ {
auto srcUri = srcStore->getUri(); auto srcUri = srcStore.getUri();
auto dstUri = dstStore->getUri(); auto dstUri = dstStore.getUri();
Activity act(*logger, lvlInfo, actCopyPath, Activity act(*logger, lvlInfo, actCopyPath,
srcUri == "local" || srcUri == "daemon" srcUri == "local" || srcUri == "daemon"
? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) ? fmt("copying path '%s' to '%s'", srcStore.printStorePath(storePath), dstUri)
: dstUri == "local" || dstUri == "daemon" : dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) ? fmt("copying path '%s' from '%s'", srcStore.printStorePath(storePath), srcUri)
: fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), : fmt("copying path '%s' from '%s' to '%s'", srcStore.printStorePath(storePath), srcUri, dstUri),
{srcStore->printStorePath(storePath), srcUri, dstUri}); {srcStore.printStorePath(storePath), srcUri, dstUri});
PushActivity pact(act.id); PushActivity pact(act.id);
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore.queryPathInfo(storePath);
uint64_t total = 0; uint64_t total = 0;
// recompute store path on the chance dstStore does it differently // recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info); auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(info->path == info2->path); assert(info->path == info2->path);
info = info2; info = info2;
} }
@ -811,17 +815,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
act.progress(total, info->narSize); act.progress(total, info->narSize);
}); });
TeeSink tee { sink, progressSink }; TeeSink tee { sink, progressSink };
srcStore->narFromPath(storePath, tee); srcStore.narFromPath(storePath, tee);
}, [&]() { }, [&]() {
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri());
}); });
dstStore->addToStore(*info, *source, repair, checkSigs); dstStore.addToStore(*info, *source, repair, checkSigs);
} }
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths, std::map<StorePath, StorePath> copyPaths(
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) Store & srcStore,
Store & dstStore,
const RealisedPath::Set & paths,
RepairFlag repair,
CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::set<Realisation> toplevelRealisations; std::set<Realisation> toplevelRealisations;
@ -839,11 +848,11 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
try { try {
// Copy the realisation closure // Copy the realisation closure
processGraph<Realisation>( processGraph<Realisation>(
pool, Realisation::closure(*srcStore, toplevelRealisations), pool, Realisation::closure(srcStore, toplevelRealisations),
[&](const Realisation & current) -> std::set<Realisation> { [&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> children; std::set<Realisation> children;
for (const auto & [drvOutput, _] : current.dependentRealisations) { for (const auto & [drvOutput, _] : current.dependentRealisations) {
auto currentChild = srcStore->queryRealisation(drvOutput); auto currentChild = srcStore.queryRealisation(drvOutput);
if (!currentChild) if (!currentChild)
throw Error( throw Error(
"incomplete realisation closure: '%s' is a " "incomplete realisation closure: '%s' is a "
@ -854,7 +863,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
return children; return children;
}, },
[&](const Realisation& current) -> void { [&](const Realisation& current) -> void {
dstStore->registerDrvOutput(current, checkSigs); dstStore.registerDrvOutput(current, checkSigs);
}); });
} catch (MissingExperimentalFeature & e) { } catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might // Don't fail if the remote doesn't support CA derivations is it might
@ -869,10 +878,15 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
return pathsMap; return pathsMap;
} }
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, std::map<StorePath, StorePath> copyPaths(
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths,
RepairFlag repair,
CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{ {
auto valid = dstStore->queryValidPaths(storePaths, substitute); auto valid = dstStore.queryValidPaths(storePaths, substitute);
StorePathSet missing; StorePathSet missing;
for (auto & path : storePaths) for (auto & path : storePaths)
@ -883,12 +897,12 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
pathsMap.insert_or_assign(path, path); pathsMap.insert_or_assign(path, path);
// FIXME: Temporary hack to copy closures in a single round-trip. // FIXME: Temporary hack to copy closures in a single round-trip.
if (dynamic_cast<RemoteStore *>(&*dstStore)) { if (dynamic_cast<RemoteStore *>(&dstStore)) {
if (!missing.empty()) { if (!missing.empty()) {
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
srcStore->exportPaths(missing, sink); srcStore.exportPaths(missing, sink);
}); });
dstStore->importPaths(*source, NoCheckSigs); dstStore.importPaths(*source, NoCheckSigs);
} }
return pathsMap; return pathsMap;
} }
@ -910,18 +924,21 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
StorePathSet(missing.begin(), missing.end()), StorePathSet(missing.begin(), missing.end()),
[&](const StorePath & storePath) { [&](const StorePath & storePath) {
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); debug("replaced path '%s' to '%s' for substituter '%s'",
srcStore.printStorePath(storePath),
dstStore.printStorePath(storePathForDst),
dstStore.getUri());
} }
pathsMap.insert_or_assign(storePath, storePathForDst); pathsMap.insert_or_assign(storePath, storePathForDst);
if (dstStore->isValidPath(storePath)) { if (dstStore.isValidPath(storePath)) {
nrDone++; nrDone++;
showProgress(); showProgress();
return StorePathSet(); return StorePathSet();
@ -936,19 +953,22 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
[&](const StorePath & storePath) { [&](const StorePath & storePath) {
checkInterrupt(); checkInterrupt();
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath; auto storePathForDst = storePath;
if (info->ca && info->references.empty()) { if (info->ca && info->references.empty()) {
storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir) if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath); assert(storePathForDst == storePath);
if (storePathForDst != storePath) if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); debug("replaced path '%s' to '%s' for substituter '%s'",
srcStore.printStorePath(storePath),
dstStore.printStorePath(storePathForDst),
dstStore.getUri());
} }
pathsMap.insert_or_assign(storePath, storePathForDst); pathsMap.insert_or_assign(storePath, storePathForDst);
if (!dstStore->isValidPath(storePathForDst)) { if (!dstStore.isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning); MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress(); showProgress();
try { try {
@ -957,7 +977,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
nrFailed++; nrFailed++;
if (!settings.keepGoing) if (!settings.keepGoing)
throw e; throw e;
logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
showProgress(); showProgress();
return; return;
} }
@ -970,17 +990,17 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
} }
void copyClosure( void copyClosure(
ref<Store> srcStore, Store & srcStore,
ref<Store> dstStore, Store & dstStore,
const RealisedPath::Set & paths, const RealisedPath::Set & paths,
RepairFlag repair, RepairFlag repair,
CheckSigsFlag checkSigs, CheckSigsFlag checkSigs,
SubstituteFlag substitute) SubstituteFlag substitute)
{ {
if (srcStore == dstStore) return; if (&srcStore == &dstStore) return;
RealisedPath::Set closure; RealisedPath::Set closure;
RealisedPath::closure(*srcStore, paths, closure); RealisedPath::closure(srcStore, paths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
} }

17
src/libstore/store-api.hh
View file

@ -751,8 +751,12 @@ protected:
/* Copy a path from one store to another. */ /* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, void copyStorePath(
const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); Store & srcStore,
Store & dstStore,
const StorePath & storePath,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
/* Copy store paths from one store to another. The paths may be copied /* Copy store paths from one store to another. The paths may be copied
@ -761,20 +765,23 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
of store paths is not automatically closed; use copyClosure() for of store paths is not automatically closed; use copyClosure() for
that. Returns a map of what each path was copied to the dstStore that. Returns a map of what each path was copied to the dstStore
as. */ as. */
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, std::map<StorePath, StorePath> copyPaths(
Store & srcStore, Store & dstStore,
const RealisedPath::Set &, const RealisedPath::Set &,
RepairFlag repair = NoRepair, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute); SubstituteFlag substitute = NoSubstitute);
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, std::map<StorePath, StorePath> copyPaths(
Store & srcStore, Store & dstStore,
const StorePathSet & paths, const StorePathSet & paths,
RepairFlag repair = NoRepair, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute); SubstituteFlag substitute = NoSubstitute);
/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ /* Copy the closure of `paths` from `srcStore` to `dstStore`. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore, void copyClosure(
Store & srcStore, Store & dstStore,
const RealisedPath::Set & paths, const RealisedPath::Set & paths,
RepairFlag repair = NoRepair, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,

4
src/nix-build/nix-build.cc
View file

@ -397,7 +397,7 @@ static void main_nix_build(int argc, char * * argv)
pathsToCopy.insert(src); pathsToCopy.insert(src);
} }
copyClosure(evalStore, store, pathsToCopy); copyClosure(*evalStore, *store, pathsToCopy);
buildPaths(pathsToBuild); buildPaths(pathsToBuild);
@ -564,7 +564,7 @@ static void main_nix_build(int argc, char * * argv)
drvMap[drvPath] = {drvMap.size(), {outputName}}; drvMap[drvPath] = {drvMap.size(), {outputName}};
} }
copyClosure(evalStore, store, drvsToCopy); copyClosure(*evalStore, *store, drvsToCopy);
buildPaths(pathsToBuild); buildPaths(pathsToBuild);

2
src/nix-copy-closure/nix-copy-closure.cc
View file

@ -54,7 +54,7 @@ static int main_nix_copy_closure(int argc, char ** argv)
for (auto & path : storePaths) for (auto & path : storePaths)
storePaths2.insert(from->followLinksToStorePath(path)); storePaths2.insert(from->followLinksToStorePath(path));
copyClosure(from, to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes); copyClosure(*from, *to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes);
return 0; return 0;
} }

2
src/nix/copy.cc
View file

@ -90,7 +90,7 @@ struct CmdCopy : BuiltPathsCommand
} }
copyPaths( copyPaths(
srcStore, dstStore, stuffToCopy, NoRepair, checkSigs, substitute); *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute);
} }
}; };

2
src/nix/flake.cc
View file

@ -841,7 +841,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
if (!dryRun && !dstUri.empty()) { if (!dryRun && !dstUri.empty()) {
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
copyPaths(store, dstStore, sources); copyPaths(*store, *dstStore, sources);
} }
} }
}; };