From 6f30245e0a0cf764740c1f0f57ad8ebd236917dd Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 23 Mar 2024 00:47:16 +0100 Subject: [PATCH] libstore: have path dump return a generator Change-Id: Ic4cf5562504aa29130304469936f958c0426e5ef --- src/libfetchers/path.cc | 6 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 11 ++- src/libstore/build/local-derivation-goal.cc | 4 +- src/libstore/daemon.cc | 2 +- src/libstore/local-fs-store.cc | 2 +- src/libstore/local-store.cc | 8 +- src/libstore/path-references.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 15 ++-- src/libutil/archive.cc | 85 ++++++++++++--------- src/libutil/archive.hh | 6 +- src/libutil/hash.cc | 2 +- src/libutil/serialise.hh | 37 +++++++++ src/libutil/source-path.hh | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/dump-path.cc | 2 +- src/nix/hash.cc | 2 +- src/nix/profile.cc | 2 +- 20 files changed, 122 insertions(+), 74 deletions(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 11c952dc2..f34010b15 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -130,10 +130,8 @@ struct PathInputScheme : InputScheme time_t mtime = 0; if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. - auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); - }); - storePath = store->addToStoreFromDump(*src, "source"); + auto src = WireSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)}; + storePath = store->addToStoreFromDump(src, "source"); } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 68cb0ca40..c63b6a06a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,7 +71,7 @@ DownloadFileResult downloadFile( storePath = std::move(cached->storePath); } else { StringSink sink; - dumpString(res.data, sink); + sink << dumpString(res.data); auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 502fac05f..d33139092 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -5,6 +5,7 @@ #include "fs-accessor.hh" #include "globals.hh" #include "nar-info.hh" +#include "serialise.hh" #include "sync.hh" #include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" @@ -413,16 +414,14 @@ StorePath BinaryCacheStore::addToStore( HashSink sink { hashAlgo }; if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); + sink << dumpPath(srcPath, filter); } else { readFile(srcPath, sink); } auto h = sink.finish().first; - auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink, filter); - }); - return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { + auto source = WireSource{dumpPath(srcPath, filter)}; + return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, @@ -455,7 +454,7 @@ StorePath BinaryCacheStore::addTextToStore( return path; StringSink sink; - dumpString(s, sink); + sink << dumpString(s); StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ad926bdd1..37c8a7708 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2393,7 +2393,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() /* FIXME: Is this actually streaming? */ auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink(rewrites, nextSink); - dumpPath(actualPath, rsink); + rsink << dumpPath(actualPath); rsink.flush(); }); Path tmpPath = actualPath + ".tmp"; @@ -2459,7 +2459,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() [&](const FileIngestionMethod & m2) { switch (m2) { case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); + caSink << dumpPath(actualPath); break; case FileIngestionMethod::Flat: readFile(actualPath, caSink); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 49b0d5e95..3c779bcd1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -867,7 +867,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + to << dumpPath(store->toRealPath(path)); break; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e9..2bf0a960c 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -82,7 +82,7 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) { if (!isValidPath(path)) throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); + sink << dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size())); } const std::string LocalFSStore::drvsLogDir = "drvs"; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1544fead9..9c86a6b88 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1395,7 +1395,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto narHash = std::pair { hash, size }; if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { HashSink narSink { htSHA256 }; - dumpPath(realPath, narSink); + narSink << dumpPath(realPath); narHash = narSink.finish(); } @@ -1450,7 +1450,7 @@ StorePath LocalStore::addTextToStore( canonicalisePathMetaData(realPath, {}); StringSink sink; - dumpString(s, sink); + sink << dumpString(s); auto narHash = hashString(htSHA256, sink.s); optimisePath(realPath, repair); @@ -1590,7 +1590,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto hashSink = HashSink(info->narHash.type); - dumpPath(Store::toRealPath(i), hashSink); + hashSink << dumpPath(Store::toRealPath(i)); auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { @@ -1891,7 +1891,7 @@ ContentAddress LocalStore::hashCAPath( [&](const FileIngestionMethod & m2) { switch (m2) { case FileIngestionMethod::Recursive: - dumpPath(path, caSink); + caSink << dumpPath(path); break; case FileIngestionMethod::Flat: readFile(path, caSink); diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..6fcb3db48 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -65,7 +65,7 @@ StorePathSet scanForReferences( TeeSink sink { refsSink, toTee }; /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); + sink << dumpPath(path); return refsSink.getResultPaths(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dd95f00e1..8ee4e9b91 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -473,7 +473,7 @@ ref RemoteStore::addCAToStore( dump.drainInto(conn->to); } else { std::string contents = dump.drain(); - dumpString(contents, conn->to); + conn->to << dumpString(contents); } } conn.processStderr(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 94bc44e35..e1575deb3 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "derivations.hh" #include "store-api.hh" +#include "serialise.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" @@ -272,7 +273,7 @@ StorePath Store::addToStore( Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); + sink << dumpPath(srcPath, filter); else readFile(srcPath, sink); }); @@ -421,13 +422,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Functionally, this means that fileSource will yield the content of srcPath. The fact that we use scratchpadSink as a temporary buffer here is an implementation detail. */ - auto fileSource = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); - }); + auto fileSource = WireSource{dumpPath(srcPath)}; /* tapped provides the same data as fileSource, but we also write all the information to narSink. */ - TeeSource tapped { *fileSource, narSink }; + TeeSource tapped { fileSource, narSink }; ParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat @@ -462,10 +461,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, info.narSize = narSize; if (!isValidPath(info.path)) { - auto source = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); - }); - addToStore(info, *source); + auto source = WireSource{dumpPath(srcPath)}; + addToStore(info, source); } return info; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 00536c1e1..223172529 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "serialise.hh" #include "util.hh" #include "config.hh" #include "signals.hh" @@ -39,10 +40,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +static WireFormatGenerator dumpContents(const Path & path, off_t size) { - sink << "contents" << size; + co_yield "contents"; + co_yield size; AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; if (!fd) throw SysError("opening file '%1%'", path); @@ -54,31 +55,35 @@ static void dumpContents(const Path & path, off_t size, auto n = std::min(left, buf.size()); readFull(fd.get(), buf.data(), n); left -= n; - sink({buf.data(), n}); + co_yield std::span{buf.data(), n}; } - writePadding(size, sink); + co_yield SerializingTransform::padding(size); } -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) +static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter) { checkInterrupt(); auto st = lstat(path); - time_t result = st.st_mtime; + mtime = st.st_mtime; - sink << "("; + co_yield "("; if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); + co_yield "type"; + co_yield "regular"; + if (st.st_mode & S_IXUSR) { + co_yield "executable"; + co_yield ""; + } + co_yield dumpContents(path, st.st_size); } else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; + co_yield "type"; + co_yield "directory"; /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ @@ -100,41 +105,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; + co_yield "entry"; + co_yield "("; + co_yield "name"; + co_yield i.first; + co_yield "node"; + time_t tmp_mtime; + co_yield dump(path + "/" + i.second, tmp_mtime, filter); + if (tmp_mtime > mtime) { + mtime = tmp_mtime; } - sink << ")"; + co_yield ")"; } } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); + else if (S_ISLNK(st.st_mode)) { + co_yield "type"; + co_yield "symlink"; + co_yield "target"; + co_yield readLink(path); + } else throw Error("file '%1%' has an unsupported type", path); - sink << ")"; - - return result; + co_yield ")"; } -time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) +WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + co_yield narVersionMagic1; + co_yield dump(path, mtime, filter); } -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +WireFormatGenerator dumpPath(const Path & path, PathFilter & filter) { - dumpPathAndGetMtime(path, sink, filter); + time_t ignored; + co_yield dumpPathAndGetMtime(path, ignored, filter); } -void dumpString(std::string_view s, Sink & sink) +WireFormatGenerator dumpString(std::string_view s) { - sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; + co_yield narVersionMagic1; + co_yield "("; + co_yield "type"; + co_yield "regular"; + co_yield "contents"; + co_yield s; + co_yield ")"; } @@ -391,10 +410,8 @@ void copyNAR(Source & source, Sink & sink) void copyPath(const Path & from, const Path & to) { - auto source = sinkToSource([&](Sink & sink) { - dumpPath(from, sink); - }); - restorePath(to, *source); + auto source = WireSource{dumpPath(from)}; + restorePath(to, source); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a41..5d9c40436 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -56,13 +56,13 @@ namespace nix { * `+` denotes string concatenation. * ``` */ -void dumpPath(const Path & path, Sink & sink, +WireFormatGenerator dumpPath(const Path & path, PathFilter & filter = defaultPathFilter); /** * Same as dumpPath(), but returns the last modified date of the path. */ -time_t dumpPathAndGetMtime(const Path & path, Sink & sink, +WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime, PathFilter & filter = defaultPathFilter); /** @@ -70,7 +70,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, * * @param s Contents of the file. */ -void dumpString(std::string_view s, Sink & sink); +WireFormatGenerator dumpString(std::string_view s); /** * \todo Fix this API, it sucks. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..5dedbc221 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -371,7 +371,7 @@ HashResult hashPath( HashType ht, const Path & path, PathFilter & filter) { HashSink sink(ht); - dumpPath(path, sink, filter); + sink << dumpPath(path, filter); return sink.finish(); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 89fff7a04..5f70a6c81 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -385,6 +385,30 @@ inline void drainGenerator(WireFormatGenerator g, std::derived_from auto & } } +struct WireSource : Source +{ + WireSource(WireFormatGenerator && g) : g(std::move(g)) {} + + virtual size_t read(char * data, size_t len) + { + while (!buf.size() && g) { + buf = g(); + } + if (!buf.size()) { + throw EndOfFile("coroutine has finished"); + } + + len = std::min(len, buf.size()); + memcpy(data, buf.data(), len); + buf = buf.subspan(len); + return len; + } + +private: + WireFormatGenerator g; + std::span buf{}; +}; + struct SerializingTransform { std::array buf; @@ -407,6 +431,19 @@ struct SerializingTransform return {buf.begin(), 8}; } + static std::span padding(size_t unpadded) + { + return std::span("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0); + } + + // opt in to generator chaining. without this co_yielding + // another generator of any type will cause a type error. + template + auto operator()(Generator, TF> && g) + { + return std::move(g); + } + // only choose this for *exactly* char spans, do not allow implicit // conversions. this would cause ambiguities with strings literals, // and resolving those with more string-like overloads needs a lot. diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 03cc998e3..df30b799f 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -99,7 +99,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { sink << nix::dumpPath(path.abs(), filter); } /** * Return the location of this path in the "real" filesystem, if diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75d8b6c6d..612069600 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -672,7 +672,7 @@ static void opDump(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); std::string path = *opArgs.begin(); - dumpPath(path, sink); + sink << dumpPath(path); sink.flush(); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99d..c92e24fb8 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -30,7 +30,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!namePart) namePart = baseNameOf(path); StringSink sink; - dumpPath(path, sink); + sink << dumpPath(path); auto narHash = hashString(htSHA256, sink.s); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b..10bf667f1 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -56,7 +56,7 @@ struct CmdDumpPath2 : Command void run() override { FdSink sink(STDOUT_FILENO); - dumpPath(path, sink); + sink << dumpPath(path); sink.flush(); } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca9345..6872dbd61 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -88,7 +88,7 @@ struct CmdHashBase : Command readFile(path, *hashSink); break; case FileIngestionMethod::Recursive: - dumpPath(path, *hashSink); + *hashSink << dumpPath(path); break; } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 59437e63e..1729d612b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -215,7 +215,7 @@ struct ProfileManifest /* Add the symlink tree to the store. */ StringSink sink; - dumpPath(tempDir, sink); + sink << dumpPath(tempDir); auto narHash = hashString(htSHA256, sink.s);