libstore: convert dumpPath to a generator

Change-Id: Ic4cf5562504aa29130304469936f958c0426e5ef

src/libcmd/cmd-profiles.cc

@@ -244,7 +244,7 @@ StorePath ProfileManifest::build(ref<Store> store)
 
     /* Add the symlink tree to the store. */
     StringSink sink;
-    dumpPath(tempDir, sink);
+    sink << dumpPath(tempDir);
 
     auto narHash = hashString(htSHA256, sink.s);
 

src/libfetchers/path.cc

@@ -130,10 +130,8 @@ struct PathInputScheme : InputScheme
         time_t mtime = 0;
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             // FIXME: try to substitute storePath.
-            auto src = sinkToSource([&](Sink & sink) {
+            auto src = GeneratorSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)};
-                mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
+            storePath = store->addToStoreFromDump(src, "source");
-            });
-            storePath = store->addToStoreFromDump(*src, "source");
         }
         input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 

src/libfetchers/tarball.cc

@@ -71,7 +71,7 @@ DownloadFileResult downloadFile(
         storePath = std::move(cached->storePath);
     } else {
         StringSink sink;
-        dumpString(res.data, sink);
+        sink << dumpString(res.data);
         auto hash = hashString(htSHA256, res.data);
         ValidPathInfo info {
             *store,

src/libstore/binary-cache-store.cc

@@ -383,16 +383,14 @@ StorePath BinaryCacheStore::addToStore(
 
     HashSink sink { hashAlgo };
     if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
+        sink << dumpPath(srcPath, filter);
     } else {
         readFileSource(srcPath)->drainInto(sink);
     }
     auto h = sink.finish().first;
 
-    auto source = sinkToSource([&](Sink & sink) {
+    auto source = GeneratorSource{dumpPath(srcPath, filter)};
-        dumpPath(srcPath, sink, filter);
+    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
-    });
-    return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
@@ -425,7 +423,7 @@ StorePath BinaryCacheStore::addTextToStore(
         return path;
 
     StringSink sink;
-    dumpString(s, sink);
+    sink << dumpString(s);
     StringSource source(sink.s);
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {

src/libstore/build/local-derivation-goal.cc

@@ -2158,7 +2158,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 /* FIXME: Is this actually streaming? */
                 auto source = sinkToSource([&](Sink & nextSink) {
                     RewritingSink rsink(rewrites, nextSink);
-                    dumpPath(actualPath, rsink);
+                    rsink << dumpPath(actualPath);
                     rsink.flush();
                 });
                 Path tmpPath = actualPath + ".tmp";
@@ -2224,7 +2224,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 [&](const FileIngestionMethod & m2) {
                     switch (m2) {
                     case FileIngestionMethod::Recursive:
-                        dumpPath(actualPath, caSink);
+                        caSink << dumpPath(actualPath);
                         break;
                     case FileIngestionMethod::Flat:
                         readFileSource(actualPath)->drainInto(caSink);

src/libstore/daemon.cc

@@ -850,7 +850,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->toRealPath(path), to);
+        to << dumpPath(store->toRealPath(path));
         break;
     }
 

src/libstore/local-fs-store.cc

@@ -82,7 +82,7 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
 {
     if (!isValidPath(path))
         throw Error("path '%s' does not exist in store", printStorePath(path));
-    dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
+    sink << dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()));
 }
 
 const std::string LocalFSStore::drvsLogDir = "drvs";

src/libstore/local-store.cc

@@ -1406,7 +1406,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
             auto narHash = std::pair { hash, size };
             if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
                 HashSink narSink { htSHA256 };
-                dumpPath(realPath, narSink);
+                narSink << dumpPath(realPath);
                 narHash = narSink.finish();
             }
 
@@ -1461,7 +1461,7 @@ StorePath LocalStore::addTextToStore(
             canonicalisePathMetaData(realPath, {});
 
             StringSink sink;
-            dumpString(s, sink);
+            sink << dumpString(s);
             auto narHash = hashString(htSHA256, sink.s);
 
             optimisePath(realPath, repair);
@@ -1601,7 +1601,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 auto hashSink = HashSink(info->narHash.type);
 
-                dumpPath(Store::toRealPath(i), hashSink);
+                hashSink << dumpPath(Store::toRealPath(i));
                 auto current = hashSink.finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
@@ -1895,7 +1895,7 @@ ContentAddress LocalStore::hashCAPath(
         [&](const FileIngestionMethod & m2) {
             switch (m2) {
             case FileIngestionMethod::Recursive:
-                dumpPath(path, caSink);
+                caSink << dumpPath(path);
                 break;
             case FileIngestionMethod::Flat:
                 readFileSource(path)->drainInto(caSink);

src/libstore/path-references.cc

@@ -61,7 +61,7 @@ StorePathSet scanForReferences(
     TeeSink sink { refsSink, toTee };
 
     /* Look for the hashes in the NAR dump of the path. */
-    dumpPath(path, sink);
+    sink << dumpPath(path);
 
     return refsSink.getResultPaths();
 }

src/libstore/remote-store.cc

@@ -422,7 +422,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                             dump.drainInto(conn->to);
                         } else {
                             std::string contents = dump.drain();
-                            dumpString(contents, conn->to);
+                            conn->to << dumpString(contents);
                         }
                     }
                     conn.processStderr();

src/libstore/store-api.cc

@@ -277,7 +277,7 @@ StorePath Store::addToStore(
     Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
         if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
+            sink << dumpPath(srcPath, filter);
         else
             readFileSource(srcPath)->drainInto(sink);
     });
@@ -425,13 +425,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     /* Functionally, this means that fileSource will yield the content of
        srcPath. The fact that we use scratchpadSink as a temporary buffer here
        is an implementation detail. */
-    auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
+    auto fileSource = GeneratorSource{dumpPath(srcPath)};
-        dumpPath(srcPath, scratchpadSink);
-    });
 
     /* tapped provides the same data as fileSource, but we also write all the
        information to narSink. */
-    TeeSource tapped { *fileSource, narSink };
+    TeeSource tapped { fileSource, narSink };
 
     ParseSink blank;
     auto & parseSink = method == FileIngestionMethod::Flat
@@ -466,10 +464,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     info.narSize = narSize;
 
     if (!isValidPath(info.path)) {
-        auto source = sinkToSource([&](Sink & scratchpadSink) {
+        auto source = GeneratorSource{dumpPath(srcPath)};
-            dumpPath(srcPath, scratchpadSink);
+        addToStore(info, source);
-        });
-        addToStore(info, *source);
     }
 
     return info;

src/libutil/archive.cc

@@ -40,10 +40,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, off_t size,
+static WireFormatGenerator dumpContents(const Path & path, off_t size)
-    Sink & sink)
 {
-    sink << "contents" << size;
+    co_yield "contents";
+    co_yield size;
 
     AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
     if (!fd) throw SysError("opening file '%1%'", path);
@@ -55,31 +55,35 @@ static void dumpContents(const Path & path, off_t size,
         auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink({buf.data(), n});
+        co_yield std::span{buf.data(), n};
     }
 
-    writePadding(size, sink);
+    co_yield SerializingTransform::padding(size);
 }
 
 
-static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
+static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter)
 {
     checkInterrupt();
 
     auto st = lstat(path);
-    time_t result = st.st_mtime;
+    mtime = st.st_mtime;
 
-    sink << "(";
+    co_yield "(";
 
     if (S_ISREG(st.st_mode)) {
-        sink << "type" << "regular";
+        co_yield "type";
-        if (st.st_mode & S_IXUSR)
+        co_yield "regular";
-            sink << "executable" << "";
+        if (st.st_mode & S_IXUSR) {
-        dumpContents(path, st.st_size, sink);
+            co_yield "executable";
+            co_yield "";
+        }
+        co_yield dumpContents(path, st.st_size);
     }
 
     else if (S_ISDIR(st.st_mode)) {
-        sink << "type" << "directory";
+        co_yield "type";
+        co_yield "directory";
 
         /* If we're on a case-insensitive system like macOS, undo
            the case hack applied by restorePath(). */
@@ -101,41 +105,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
 
         for (auto & i : unhacked)
             if (filter(path + "/" + i.first)) {
-                sink << "entry" << "(" << "name" << i.first << "node";
+                co_yield "entry";
-                auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
+                co_yield "(";
-                if (tmp_mtime > result) {
+                co_yield "name";
-                    result = tmp_mtime;
+                co_yield i.first;
+                co_yield "node";
+                time_t tmp_mtime;
+                co_yield dump(path + "/" + i.second, tmp_mtime, filter);
+                if (tmp_mtime > mtime) {
+                    mtime = tmp_mtime;
                 }
-                sink << ")";
+                co_yield ")";
             }
     }
 
-    else if (S_ISLNK(st.st_mode))
+    else if (S_ISLNK(st.st_mode)) {
-        sink << "type" << "symlink" << "target" << readLink(path);
+        co_yield "type";
+        co_yield "symlink";
+        co_yield "target";
+        co_yield readLink(path);
+    }
 
     else throw Error("file '%1%' has an unsupported type", path);
 
-    sink << ")";
+    co_yield ")";
-
-    return result;
 }
 
 
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime, PathFilter & filter)
 {
-    sink << narVersionMagic1;
+    co_yield narVersionMagic1;
-    return dump(path, sink, filter);
+    co_yield dump(path, mtime, filter);
 }
 
-void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
+WireFormatGenerator dumpPath(Path path, PathFilter & filter)
 {
-    dumpPathAndGetMtime(path, sink, filter);
+    time_t ignored;
+    co_yield dumpPathAndGetMtime(path, ignored, filter);
 }
 
 
-void dumpString(std::string_view s, Sink & sink)
+WireFormatGenerator dumpString(std::string_view s)
 {
-    sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
+    co_yield narVersionMagic1;
+    co_yield "(";
+    co_yield "type";
+    co_yield "regular";
+    co_yield "contents";
+    co_yield s;
+    co_yield ")";
 }
 
 

src/libutil/archive.hh

@@ -57,13 +57,13 @@ namespace nix {
  *   `+` denotes string concatenation.
  * ```
  */
-void dumpPath(const Path & path, Sink & sink,
+WireFormatGenerator dumpPath(Path path,
     PathFilter & filter = defaultPathFilter);
 
 /**
  * Same as dumpPath(), but returns the last modified date of the path.
  */
-time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
+WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime,
     PathFilter & filter = defaultPathFilter);
 
 /**
@@ -71,7 +71,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  *
  * @param s Contents of the file.
  */
-void dumpString(std::string_view s, Sink & sink);
+WireFormatGenerator dumpString(std::string_view s);
 
 /**
  * \todo Fix this API, it sucks.

src/libutil/hash.cc

@@ -370,7 +370,7 @@ HashResult hashPath(
     HashType ht, const Path & path, PathFilter & filter)
 {
     HashSink sink(ht);
-    dumpPath(path, sink, filter);
+    sink << dumpPath(path, filter);
     return sink.finish();
 }
 

src/libutil/source-path.hh

@@ -101,7 +101,7 @@ struct SourcePath
     void dumpPath(
         Sink & sink,
         PathFilter & filter = defaultPathFilter) const
-    { return nix::dumpPath(path.abs(), sink, filter); }
+    { sink << nix::dumpPath(path.abs(), filter); }
 
     /**
      * Return the location of this path in the "real" filesystem, if

src/nix-store/nix-store.cc

@@ -671,7 +671,7 @@ static void opDump(Strings opFlags, Strings opArgs)
 
     FdSink sink(STDOUT_FILENO);
     std::string path = *opArgs.begin();
-    dumpPath(path, sink);
+    sink << dumpPath(path);
     sink.flush();
 }
 

src/nix/add-to-store.cc

@@ -30,7 +30,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         if (!namePart) namePart = baseNameOf(path);
 
         StringSink sink;
-        dumpPath(path, sink);
+        sink << dumpPath(path);
 
         auto narHash = hashString(htSHA256, sink.s);
 

src/nix/dump-path.cc

@@ -58,7 +58,7 @@ struct CmdDumpPath2 : Command
     {
         logger->pause();
         FdSink sink(STDOUT_FILENO);
-        dumpPath(path, sink);
+        sink << dumpPath(path);
         sink.flush();
     }
 };

src/nix/hash.cc

@@ -88,7 +88,7 @@ struct CmdHashBase : Command
                 readFileSource(path)->drainInto(*hashSink);
                 break;
             case FileIngestionMethod::Recursive:
-                dumpPath(path, *hashSink);
+                *hashSink << dumpPath(path);
                 break;
             }