From cb0553ecd0122f693653c1ac82beb26d127ce3cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Wed, 8 Jun 2022 14:03:46 +0200
Subject: [PATCH] Restore the "low-latency" ssh copying

---
 src/libstore/remote-store.cc | 17 +++++++++++++
 src/libstore/remote-store.hh |  8 ++++++
 src/libstore/store-api.cc    | 49 ++++++++++++++++--------------------
 src/libstore/store-api.hh    |  6 ++++-
 4 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index bc36aef5d..ad2e5c18a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
+void RemoteStore::addMultipleToStore(
+    PathsSource & pathsToCopy,
+    Activity & act,
+    RepairFlag repair,
+    CheckSigsFlag checkSigs)
+{
+    auto source = sinkToSource([&](Sink & sink) {
+        sink << pathsToCopy.size();
+        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+            pathInfo.write(sink, *this, 16);
+            pathSource->drainInto(sink);
+        }
+    });
+
+    addMultipleToStore(*source, repair, checkSigs);
+}
+
 void RemoteStore::addMultipleToStore(
     Source & source,
     RepairFlag repair,
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 8493be6fc..5a599997e 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -88,6 +88,14 @@ public:
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;
 
+    void addMultipleToStore(
+    PathsSource & pathsToCopy,
+    Activity & act,
+    RepairFlag repair,
+    CheckSigsFlag checkSigs) override;
+
+
+
     StorePath addTextToStore(
         std::string_view name,
         std::string_view s,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 008451666..eeec6c6f7 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -259,7 +259,7 @@ StorePath Store::addToStore(
 }
 
 void Store::addMultipleToStore(
-    std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> & pathsToCopy,
+    PathsSource & pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -1072,37 +1072,33 @@ std::map<StorePath, StorePath> copyPaths(
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
+    Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+
+    // In the general case, `addMultipleToStore` requires a sorted list of
+    // store paths to add, so sort them right now
+    auto sortedMissing = srcStore.topoSortPaths(missing);
+    std::reverse(sortedMissing.begin(), sortedMissing.end());
+
     std::map<StorePath, StorePath> pathsMap;
     for (auto & path : storePaths)
         pathsMap.insert_or_assign(path, path);
 
-    Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+    Store::PathsSource pathsToCopy;
 
-    /* auto sorted = srcStore.topoSortPaths(missing); */
-    /* std::reverse(sorted.begin(), sorted.end()); */
-
-    /* auto source = sinkToSource([&](Sink & sink) { */
-    /*     sink << sorted.size(); */
-    /*     for (auto & storePath : sorted) { */
-    /*         auto srcUri = srcStore.getUri(); */
-    /*         auto dstUri = dstStore.getUri(); */
-    /*         auto storePathS = srcStore.printStorePath(storePath); */
-    /*         Activity act(*logger, lvlInfo, actCopyPath, */
-    /*             makeCopyPathMessage(srcUri, dstUri, storePathS), */
-    /*             {storePathS, srcUri, dstUri}); */
-    /*         PushActivity pact(act.id); */
-
-    /*         auto info = srcStore.queryPathInfo(storePath); */
-    /*         info->write(sink, srcStore, 16); */
-    /*         srcStore.narFromPath(storePath, sink); */
-    /*     } */
-    /* }); */
-
-    std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> pathsToCopy;
-
-    for (auto & missingPath : missing) {
+    for (auto & missingPath : sortedMissing) {
         auto info = srcStore.queryPathInfo(missingPath);
         auto source = sinkToSource([&](Sink & sink) {
+
+            // We can reasonably assume that the copy will happen whenever we
+            // read the path, so log something about that at that point
+            auto srcUri = srcStore.getUri();
+            auto dstUri = dstStore.getUri();
+            auto storePathS = srcStore.printStorePath(missingPath);
+            Activity act(*logger, lvlInfo, actCopyPath,
+                makeCopyPathMessage(srcUri, dstUri, storePathS),
+                {storePathS, srcUri, dstUri});
+            PushActivity pact(act.id);
+
             srcStore.narFromPath(missingPath, sink);
         });
         pathsToCopy.push_back(std::pair{*info, std::move(source)});
@@ -1110,9 +1106,6 @@ std::map<StorePath, StorePath> copyPaths(
 
     dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
 
-    #if 0
-    #endif
-
     return pathsMap;
 }
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d934979cf..c0a61115b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -360,13 +360,17 @@ public:
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
 
+    // A list of paths infos along with a source providing the content of the
+    // associated store path
+    using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
+
     /* Import multiple paths into the store. */
     virtual void addMultipleToStore(
         Source & source,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);
     virtual void addMultipleToStore(
-        std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>> & pathsToCopy,
+        PathsSource & pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs