kloenk/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Restore the "low-latency" ssh copying

Browse source
This commit is contained in:
Théophane Hufschmitt 2022-06-08 14:03:46 +02:00
parent 95f47c28fb
commit cb0553ecd0
4 changed files with 51 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/libstore/remote-store.cc
View file

@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
}
void RemoteStore::addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) {
pathInfo.write(sink, *this, 16);
pathSource->drainInto(sink);
}
});
addMultipleToStore(*source, repair, checkSigs);
}
void RemoteStore::addMultipleToStore(
Source & source,
RepairFlag repair,

8
src/libstore/remote-store.hh
View file

@ -88,6 +88,14 @@ public:
RepairFlag repair,
CheckSigsFlag checkSigs) override;
void addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,

49
src/libstore/store-api.cc
View file

@ -259,7 +259,7 @@ StorePath Store::addToStore(
}
void Store::addMultipleToStore(
std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> & pathsToCopy,
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
@ -1072,37 +1072,33 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(path);
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
// In the general case, `addMultipleToStore` requires a sorted list of
// store paths to add, so sort them right now
auto sortedMissing = srcStore.topoSortPaths(missing);
std::reverse(sortedMissing.begin(), sortedMissing.end());
std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
Store::PathsSource pathsToCopy;
/* auto sorted = srcStore.topoSortPaths(missing); */
/* std::reverse(sorted.begin(), sorted.end()); */
/* auto source = sinkToSource([&](Sink & sink) { */
/* sink << sorted.size(); */
/* for (auto & storePath : sorted) { */
/* auto srcUri = srcStore.getUri(); */
/* auto dstUri = dstStore.getUri(); */
/* auto storePathS = srcStore.printStorePath(storePath); */
/* Activity act(*logger, lvlInfo, actCopyPath, */
/* makeCopyPathMessage(srcUri, dstUri, storePathS), */
/* {storePathS, srcUri, dstUri}); */
/* PushActivity pact(act.id); */
/* auto info = srcStore.queryPathInfo(storePath); */
/* info->write(sink, srcStore, 16); */
/* srcStore.narFromPath(storePath, sink); */
/* } */
/* }); */
std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> pathsToCopy;
for (auto & missingPath : missing) {
for (auto & missingPath : sortedMissing) {
auto info = srcStore.queryPathInfo(missingPath);
auto source = sinkToSource([&](Sink & sink) {
// We can reasonably assume that the copy will happen whenever we
// read the path, so log something about that at that point
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
auto storePathS = srcStore.printStorePath(missingPath);
Activity act(*logger, lvlInfo, actCopyPath,
makeCopyPathMessage(srcUri, dstUri, storePathS),
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
srcStore.narFromPath(missingPath, sink);
});
pathsToCopy.push_back(std::pair{*info, std::move(source)});
@ -1110,9 +1106,6 @@ std::map<StorePath, StorePath> copyPaths(
dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
#if 0
#endif
return pathsMap;
}

6
src/libstore/store-api.hh
View file

@ -360,13 +360,17 @@ public:
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
// A list of paths infos along with a source providing the content of the
// associated store path
using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
/* Import multiple paths into the store. */
virtual void addMultipleToStore(
Source & source,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
virtual void addMultipleToStore(
std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> & pathsToCopy,
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs