libstore: have path dump return a generator

Change-Id: Ic4cf5562504aa29130304469936f958c0426e5ef

src/libfetchers/path.cc

@@ -116,10 +116,8 @@ struct PathInputScheme : InputScheme
         time_t mtime = 0;
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             // FIXME: try to substitute storePath.
-            auto src = sinkToSource([&](Sink & sink) {
+            auto src = WireSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)};
-                mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
+            storePath = store->addToStoreFromDump(src, "source");
-            });
-            storePath = store->addToStoreFromDump(*src, "source");
         }
         input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 

src/libfetchers/tarball.cc

@@ -71,7 +71,7 @@ DownloadFileResult downloadFile(
         storePath = std::move(cached->storePath);
     } else {
         StringSink sink;
-        dumpString(res.data, sink);
+        sink << dumpString(res.data);
         auto hash = hashString(htSHA256, res.data);
         ValidPathInfo info {
             *store,

src/libstore/binary-cache-store.cc

@@ -5,6 +5,7 @@
 #include "fs-accessor.hh"
 #include "globals.hh"
 #include "nar-info.hh"
+#include "serialise.hh"
 #include "sync.hh"
 #include "remote-fs-accessor.hh"
 #include "nar-info-disk-cache.hh"
@@ -413,16 +414,14 @@ StorePath BinaryCacheStore::addToStore(
 
     HashSink sink { hashAlgo };
     if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
+        sink << dumpPath(srcPath, filter);
     } else {
         readFile(srcPath, sink);
     }
     auto h = sink.finish().first;
 
-    auto source = sinkToSource([&](Sink & sink) {
+    auto source = WireSource{dumpPath(srcPath, filter)};
-        dumpPath(srcPath, sink, filter);
+    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
-    });
-    return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
@@ -455,7 +454,7 @@ StorePath BinaryCacheStore::addTextToStore(
         return path;
 
     StringSink sink;
-    dumpString(s, sink);
+    sink << dumpString(s);
     StringSource source(sink.s);
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {

src/libstore/build/local-derivation-goal.cc

@@ -2393,7 +2393,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 /* FIXME: Is this actually streaming? */
                 auto source = sinkToSource([&](Sink & nextSink) {
                     RewritingSink rsink(rewrites, nextSink);
-                    dumpPath(actualPath, rsink);
+                    rsink << dumpPath(actualPath);
                     rsink.flush();
                 });
                 Path tmpPath = actualPath + ".tmp";
@@ -2459,7 +2459,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 [&](const FileIngestionMethod & m2) {
                     switch (m2) {
                     case FileIngestionMethod::Recursive:
-                        dumpPath(actualPath, caSink);
+                        caSink << dumpPath(actualPath);
                         break;
                     case FileIngestionMethod::Flat:
                         readFile(actualPath, caSink);

src/libstore/daemon.cc

@@ -869,7 +869,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->toRealPath(path), to);
+        to << dumpPath(store->toRealPath(path));
         break;
     }
 

src/libstore/local-fs-store.cc

@@ -82,7 +82,7 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
 {
     if (!isValidPath(path))
         throw Error("path '%s' is not valid", printStorePath(path));
-    dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
+    sink << dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()));
 }
 
 const std::string LocalFSStore::drvsLogDir = "drvs";

src/libstore/local-store.cc

@@ -1395,7 +1395,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
             auto narHash = std::pair { hash, size };
             if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
                 HashSink narSink { htSHA256 };
-                dumpPath(realPath, narSink);
+                narSink << dumpPath(realPath);
                 narHash = narSink.finish();
             }
 
@@ -1450,7 +1450,7 @@ StorePath LocalStore::addTextToStore(
             canonicalisePathMetaData(realPath, {});
 
             StringSink sink;
-            dumpString(s, sink);
+            sink << dumpString(s);
             auto narHash = hashString(htSHA256, sink.s);
 
             optimisePath(realPath, repair);
@@ -1590,7 +1590,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 auto hashSink = HashSink(info->narHash.type);
 
-                dumpPath(Store::toRealPath(i), hashSink);
+                hashSink << dumpPath(Store::toRealPath(i));
                 auto current = hashSink.finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
@@ -1891,7 +1891,7 @@ ContentAddress LocalStore::hashCAPath(
         [&](const FileIngestionMethod & m2) {
             switch (m2) {
             case FileIngestionMethod::Recursive:
-                dumpPath(path, caSink);
+                caSink << dumpPath(path);
                 break;
             case FileIngestionMethod::Flat:
                 readFile(path, caSink);

src/libstore/path-references.cc

@@ -65,7 +65,7 @@ StorePathSet scanForReferences(
     TeeSink sink { refsSink, toTee };
 
     /* Look for the hashes in the NAR dump of the path. */
-    dumpPath(path, sink);
+    sink << dumpPath(path);
 
     return refsSink.getResultPaths();
 }

src/libstore/remote-store.cc

@@ -482,7 +482,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                             dump.drainInto(conn->to);
                         } else {
                             std::string contents = dump.drain();
-                            dumpString(contents, conn->to);
+                            conn->to << dumpString(contents);
                         }
                     }
                     conn.processStderr();

src/libstore/store-api.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "serialise.hh"
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
@@ -272,7 +273,7 @@ StorePath Store::addToStore(
     Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
         if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
+            sink << dumpPath(srcPath, filter);
         else
             readFile(srcPath, sink);
     });
@@ -421,13 +422,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     /* Functionally, this means that fileSource will yield the content of
        srcPath. The fact that we use scratchpadSink as a temporary buffer here
        is an implementation detail. */
-    auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
+    auto fileSource = WireSource{dumpPath(srcPath)};
-        dumpPath(srcPath, scratchpadSink);
-    });
 
     /* tapped provides the same data as fileSource, but we also write all the
        information to narSink. */
-    TeeSource tapped { *fileSource, narSink };
+    TeeSource tapped { fileSource, narSink };
 
     ParseSink blank;
     auto & parseSink = method == FileIngestionMethod::Flat
@@ -462,10 +461,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     info.narSize = narSize;
 
     if (!isValidPath(info.path)) {
-        auto source = sinkToSource([&](Sink & scratchpadSink) {
+        auto source = WireSource{dumpPath(srcPath)};
-            dumpPath(srcPath, scratchpadSink);
+        addToStore(info, source);
-        });
-        addToStore(info, *source);
     }
 
     return info;

src/libutil/archive.cc

@@ -12,6 +12,7 @@
 #include <fcntl.h>
 
 #include "archive.hh"
+#include "serialise.hh"
 #include "util.hh"
 #include "config.hh"
 #include "signals.hh"
@@ -39,10 +40,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, off_t size,
+static WireFormatGenerator dumpContents(const Path & path, off_t size)
-    Sink & sink)
 {
-    sink << "contents" << size;
+    co_yield "contents";
+    co_yield size;
 
     AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
     if (!fd) throw SysError("opening file '%1%'", path);
@@ -54,31 +55,35 @@ static void dumpContents(const Path & path, off_t size,
         auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink({buf.data(), n});
+        co_yield std::span{buf.data(), n};
     }
 
-    writePadding(size, sink);
+    co_yield SerializingTransform::padding(size);
 }
 
 
-static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
+static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter)
 {
     checkInterrupt();
 
     auto st = lstat(path);
-    time_t result = st.st_mtime;
+    mtime = st.st_mtime;
 
-    sink << "(";
+    co_yield "(";
 
     if (S_ISREG(st.st_mode)) {
-        sink << "type" << "regular";
+        co_yield "type";
-        if (st.st_mode & S_IXUSR)
+        co_yield "regular";
-            sink << "executable" << "";
+        if (st.st_mode & S_IXUSR) {
-        dumpContents(path, st.st_size, sink);
+            co_yield "executable";
+            co_yield "";
+        }
+        co_yield dumpContents(path, st.st_size);
     }
 
     else if (S_ISDIR(st.st_mode)) {
-        sink << "type" << "directory";
+        co_yield "type";
+        co_yield "directory";
 
         /* If we're on a case-insensitive system like macOS, undo
            the case hack applied by restorePath(). */
@@ -100,41 +105,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
 
         for (auto & i : unhacked)
             if (filter(path + "/" + i.first)) {
-                sink << "entry" << "(" << "name" << i.first << "node";
+                co_yield "entry";
-                auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
+                co_yield "(";
-                if (tmp_mtime > result) {
+                co_yield "name";
-                    result = tmp_mtime;
+                co_yield i.first;
+                co_yield "node";
+                time_t tmp_mtime;
+                co_yield dump(path + "/" + i.second, tmp_mtime, filter);
+                if (tmp_mtime > mtime) {
+                    mtime = tmp_mtime;
                 }
-                sink << ")";
+                co_yield ")";
             }
     }
 
-    else if (S_ISLNK(st.st_mode))
+    else if (S_ISLNK(st.st_mode)) {
-        sink << "type" << "symlink" << "target" << readLink(path);
+        co_yield "type";
+        co_yield "symlink";
+        co_yield "target";
+        co_yield readLink(path);
+    }
 
     else throw Error("file '%1%' has an unsupported type", path);
 
-    sink << ")";
+    co_yield ")";
-
-    return result;
 }
 
 
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime, PathFilter & filter)
 {
-    sink << narVersionMagic1;
+    co_yield narVersionMagic1;
-    return dump(path, sink, filter);
+    co_yield dump(path, mtime, filter);
 }
 
-void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPath(const Path & path, PathFilter & filter)
 {
-    dumpPathAndGetMtime(path, sink, filter);
+    time_t ignored;
+    co_yield dumpPathAndGetMtime(path, ignored, filter);
 }
 
 
-void dumpString(std::string_view s, Sink & sink)
+WireFormatGenerator dumpString(std::string_view s)
 {
-    sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
+    co_yield narVersionMagic1;
+    co_yield "(";
+    co_yield "type";
+    co_yield "regular";
+    co_yield "contents";
+    co_yield s;
+    co_yield ")";
 }
 
 
@@ -391,10 +410,8 @@ void copyNAR(Source & source, Sink & sink)
 
 void copyPath(const Path & from, const Path & to)
 {
-    auto source = sinkToSource([&](Sink & sink) {
+    auto source = WireSource{dumpPath(from)};
-        dumpPath(from, sink);
+    restorePath(to, source);
-    });
-    restorePath(to, *source);
 }
 
 

src/libutil/archive.hh

@@ -56,13 +56,13 @@ namespace nix {
  *   `+` denotes string concatenation.
  * ```
  */
-void dumpPath(const Path & path, Sink & sink,
+WireFormatGenerator dumpPath(const Path & path,
     PathFilter & filter = defaultPathFilter);
 
 /**
  * Same as dumpPath(), but returns the last modified date of the path.
  */
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
+WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime,
     PathFilter & filter = defaultPathFilter);
 
 /**
@@ -70,7 +70,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  *
  * @param s Contents of the file.
  */
-void dumpString(std::string_view s, Sink & sink);
+WireFormatGenerator dumpString(std::string_view s);
 
 /**
  * \todo Fix this API, it sucks.

src/libutil/hash.cc

@@ -371,7 +371,7 @@ HashResult hashPath(
     HashType ht, const Path & path, PathFilter & filter)
 {
     HashSink sink(ht);
-    dumpPath(path, sink, filter);
+    sink << dumpPath(path, filter);
     return sink.finish();
 }
 

src/libutil/serialise.hh

@@ -385,6 +385,30 @@ inline void drainGenerator(WireFormatGenerator g, std::derived_from<Sink> auto &
     }
 }
 
+struct WireSource : Source
+{
+    WireSource(WireFormatGenerator && g) : g(std::move(g)) {}
+
+    virtual size_t read(char * data, size_t len)
+    {
+        while (!buf.size() && g) {
+            buf = g();
+        }
+        if (!buf.size()) {
+            throw EndOfFile("coroutine has finished");
+        }
+
+        len = std::min(len, buf.size());
+        memcpy(data, buf.data(), len);
+        buf = buf.subspan(len);
+        return len;
+    }
+
+private:
+    WireFormatGenerator g;
+    std::span<const char> buf{};
+};
+
 struct SerializingTransform
 {
     std::array<char, 8> buf;
@@ -407,6 +431,19 @@ struct SerializingTransform
         return {buf.begin(), 8};
     }
 
+    static std::span<const char> padding(size_t unpadded)
+    {
+        return std::span("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0);
+    }
+
+    // opt in to generator chaining. without this co_yielding
+    // another generator of any type will cause a type error.
+    template<typename TF>
+    auto operator()(Generator<std::span<const char>, TF> && g)
+    {
+        return std::move(g);
+    }
+
     // only choose this for *exactly* char spans, do not allow implicit
     // conversions. this would cause ambiguities with strings literals,
     // and resolving those with more string-like overloads needs a lot.

src/libutil/source-path.hh

@@ -99,7 +99,7 @@ struct SourcePath
     void dumpPath(
         Sink & sink,
         PathFilter & filter = defaultPathFilter) const
-    { return nix::dumpPath(path.abs(), sink, filter); }
+    { sink << nix::dumpPath(path.abs(), filter); }
 
     /**
      * Return the location of this path in the "real" filesystem, if

src/nix-store/nix-store.cc

@@ -672,7 +672,7 @@ static void opDump(Strings opFlags, Strings opArgs)
 
     FdSink sink(STDOUT_FILENO);
     std::string path = *opArgs.begin();
-    dumpPath(path, sink);
+    sink << dumpPath(path);
     sink.flush();
 }
 

src/nix/add-to-store.cc

@@ -30,7 +30,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         if (!namePart) namePart = baseNameOf(path);
 
         StringSink sink;
-        dumpPath(path, sink);
+        sink << dumpPath(path);
 
         auto narHash = hashString(htSHA256, sink.s);
 

src/nix/dump-path.cc

@@ -56,7 +56,7 @@ struct CmdDumpPath2 : Command
     void run() override
     {
         FdSink sink(STDOUT_FILENO);
-        dumpPath(path, sink);
+        sink << dumpPath(path);
         sink.flush();
     }
 };

src/nix/hash.cc

@@ -88,7 +88,7 @@ struct CmdHashBase : Command
                 readFile(path, *hashSink);
                 break;
             case FileIngestionMethod::Recursive:
-                dumpPath(path, *hashSink);
+                *hashSink << dumpPath(path);
                 break;
             }
 

src/nix/profile.cc

@@ -215,7 +215,7 @@ struct ProfileManifest
 
         /* Add the symlink tree to the store. */
         StringSink sink;
-        dumpPath(tempDir, sink);
+        sink << dumpPath(tempDir);
 
         auto narHash = hashString(htSHA256, sink.s);