libstore: convert dumpPath to a generator

Change-Id: Ic4cf5562504aa29130304469936f958c0426e5ef
This commit is contained in:
eldritch horrors 2024-03-23 00:47:16 +01:00
parent b51ea465de
commit 06220a71c1
19 changed files with 80 additions and 70 deletions

View file

@ -244,7 +244,7 @@ StorePath ProfileManifest::build(ref<Store> store)
/* Add the symlink tree to the store. */ /* Add the symlink tree to the store. */
StringSink sink; StringSink sink;
dumpPath(tempDir, sink); sink << dumpPath(tempDir);
auto narHash = hashString(htSHA256, sink.s); auto narHash = hashString(htSHA256, sink.s);

View file

@ -130,10 +130,8 @@ struct PathInputScheme : InputScheme
time_t mtime = 0; time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
// FIXME: try to substitute storePath. // FIXME: try to substitute storePath.
auto src = sinkToSource([&](Sink & sink) { auto src = GeneratorSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)};
mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); storePath = store->addToStoreFromDump(src, "source");
});
storePath = store->addToStoreFromDump(*src, "source");
} }
input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); input.attrs.insert_or_assign("lastModified", uint64_t(mtime));

View file

@ -71,7 +71,7 @@ DownloadFileResult downloadFile(
storePath = std::move(cached->storePath); storePath = std::move(cached->storePath);
} else { } else {
StringSink sink; StringSink sink;
dumpString(res.data, sink); sink << dumpString(res.data);
auto hash = hashString(htSHA256, res.data); auto hash = hashString(htSHA256, res.data);
ValidPathInfo info { ValidPathInfo info {
*store, *store,

View file

@ -383,16 +383,14 @@ StorePath BinaryCacheStore::addToStore(
HashSink sink { hashAlgo }; HashSink sink { hashAlgo };
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter); sink << dumpPath(srcPath, filter);
} else { } else {
readFileSource(srcPath)->drainInto(sink); readFileSource(srcPath)->drainInto(sink);
} }
auto h = sink.finish().first; auto h = sink.finish().first;
auto source = sinkToSource([&](Sink & sink) { auto source = GeneratorSource{dumpPath(srcPath, filter)};
dumpPath(srcPath, sink, filter); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
*this, *this,
name, name,
@ -425,7 +423,7 @@ StorePath BinaryCacheStore::addTextToStore(
return path; return path;
StringSink sink; StringSink sink;
dumpString(s, sink); sink << dumpString(s);
StringSource source(sink.s); StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {

View file

@ -2158,7 +2158,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
/* FIXME: Is this actually streaming? */ /* FIXME: Is this actually streaming? */
auto source = sinkToSource([&](Sink & nextSink) { auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink(rewrites, nextSink); RewritingSink rsink(rewrites, nextSink);
dumpPath(actualPath, rsink); rsink << dumpPath(actualPath);
rsink.flush(); rsink.flush();
}); });
Path tmpPath = actualPath + ".tmp"; Path tmpPath = actualPath + ".tmp";
@ -2224,7 +2224,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
[&](const FileIngestionMethod & m2) { [&](const FileIngestionMethod & m2) {
switch (m2) { switch (m2) {
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink); caSink << dumpPath(actualPath);
break; break;
case FileIngestionMethod::Flat: case FileIngestionMethod::Flat:
readFileSource(actualPath)->drainInto(caSink); readFileSource(actualPath)->drainInto(caSink);

View file

@ -850,7 +850,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
logger->stopWork(); logger->stopWork();
dumpPath(store->toRealPath(path), to); to << dumpPath(store->toRealPath(path));
break; break;
} }

View file

@ -82,7 +82,7 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
{ {
if (!isValidPath(path)) if (!isValidPath(path))
throw Error("path '%s' does not exist in store", printStorePath(path)); throw Error("path '%s' does not exist in store", printStorePath(path));
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); sink << dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()));
} }
const std::string LocalFSStore::drvsLogDir = "drvs"; const std::string LocalFSStore::drvsLogDir = "drvs";

View file

@ -1406,7 +1406,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto narHash = std::pair { hash, size }; auto narHash = std::pair { hash, size };
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
HashSink narSink { htSHA256 }; HashSink narSink { htSHA256 };
dumpPath(realPath, narSink); narSink << dumpPath(realPath);
narHash = narSink.finish(); narHash = narSink.finish();
} }
@ -1461,7 +1461,7 @@ StorePath LocalStore::addTextToStore(
canonicalisePathMetaData(realPath, {}); canonicalisePathMetaData(realPath, {});
StringSink sink; StringSink sink;
dumpString(s, sink); sink << dumpString(s);
auto narHash = hashString(htSHA256, sink.s); auto narHash = hashString(htSHA256, sink.s);
optimisePath(realPath, repair); optimisePath(realPath, repair);
@ -1601,7 +1601,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
auto hashSink = HashSink(info->narHash.type); auto hashSink = HashSink(info->narHash.type);
dumpPath(Store::toRealPath(i), hashSink); hashSink << dumpPath(Store::toRealPath(i));
auto current = hashSink.finish(); auto current = hashSink.finish();
if (info->narHash != nullHash && info->narHash != current.first) { if (info->narHash != nullHash && info->narHash != current.first) {
@ -1895,7 +1895,7 @@ ContentAddress LocalStore::hashCAPath(
[&](const FileIngestionMethod & m2) { [&](const FileIngestionMethod & m2) {
switch (m2) { switch (m2) {
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
dumpPath(path, caSink); caSink << dumpPath(path);
break; break;
case FileIngestionMethod::Flat: case FileIngestionMethod::Flat:
readFileSource(path)->drainInto(caSink); readFileSource(path)->drainInto(caSink);

View file

@ -61,7 +61,7 @@ StorePathSet scanForReferences(
TeeSink sink { refsSink, toTee }; TeeSink sink { refsSink, toTee };
/* Look for the hashes in the NAR dump of the path. */ /* Look for the hashes in the NAR dump of the path. */
dumpPath(path, sink); sink << dumpPath(path);
return refsSink.getResultPaths(); return refsSink.getResultPaths();
} }

View file

@ -422,7 +422,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
dump.drainInto(conn->to); dump.drainInto(conn->to);
} else { } else {
std::string contents = dump.drain(); std::string contents = dump.drain();
dumpString(contents, conn->to); conn->to << dumpString(contents);
} }
} }
conn.processStderr(); conn.processStderr();

View file

@ -277,7 +277,7 @@ StorePath Store::addToStore(
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive) if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter); sink << dumpPath(srcPath, filter);
else else
readFileSource(srcPath)->drainInto(sink); readFileSource(srcPath)->drainInto(sink);
}); });
@ -425,13 +425,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
/* Functionally, this means that fileSource will yield the content of /* Functionally, this means that fileSource will yield the content of
srcPath. The fact that we use scratchpadSink as a temporary buffer here srcPath. The fact that we use scratchpadSink as a temporary buffer here
is an implementation detail. */ is an implementation detail. */
auto fileSource = sinkToSource([&](Sink & scratchpadSink) { auto fileSource = GeneratorSource{dumpPath(srcPath)};
dumpPath(srcPath, scratchpadSink);
});
/* tapped provides the same data as fileSource, but we also write all the /* tapped provides the same data as fileSource, but we also write all the
information to narSink. */ information to narSink. */
TeeSource tapped { *fileSource, narSink }; TeeSource tapped { fileSource, narSink };
ParseSink blank; ParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat auto & parseSink = method == FileIngestionMethod::Flat
@ -466,10 +464,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
info.narSize = narSize; info.narSize = narSize;
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = GeneratorSource{dumpPath(srcPath)};
dumpPath(srcPath, scratchpadSink); addToStore(info, source);
});
addToStore(info, *source);
} }
return info; return info;

View file

@ -40,10 +40,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
PathFilter defaultPathFilter = [](const Path &) { return true; }; PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, off_t size, static WireFormatGenerator dumpContents(const Path & path, off_t size)
Sink & sink)
{ {
sink << "contents" << size; co_yield "contents";
co_yield size;
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
if (!fd) throw SysError("opening file '%1%'", path); if (!fd) throw SysError("opening file '%1%'", path);
@ -55,31 +55,35 @@ static void dumpContents(const Path & path, off_t size,
auto n = std::min(left, buf.size()); auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n); readFull(fd.get(), buf.data(), n);
left -= n; left -= n;
sink({buf.data(), n}); co_yield std::span{buf.data(), n};
} }
writePadding(size, sink); co_yield SerializingTransform::padding(size);
} }
static time_t dump(const Path & path, Sink & sink, PathFilter & filter) static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter)
{ {
checkInterrupt(); checkInterrupt();
auto st = lstat(path); auto st = lstat(path);
time_t result = st.st_mtime; mtime = st.st_mtime;
sink << "("; co_yield "(";
if (S_ISREG(st.st_mode)) { if (S_ISREG(st.st_mode)) {
sink << "type" << "regular"; co_yield "type";
if (st.st_mode & S_IXUSR) co_yield "regular";
sink << "executable" << ""; if (st.st_mode & S_IXUSR) {
dumpContents(path, st.st_size, sink); co_yield "executable";
co_yield "";
}
co_yield dumpContents(path, st.st_size);
} }
else if (S_ISDIR(st.st_mode)) { else if (S_ISDIR(st.st_mode)) {
sink << "type" << "directory"; co_yield "type";
co_yield "directory";
/* If we're on a case-insensitive system like macOS, undo /* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */ the case hack applied by restorePath(). */
@ -101,41 +105,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
for (auto & i : unhacked) for (auto & i : unhacked)
if (filter(path + "/" + i.first)) { if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node"; co_yield "entry";
auto tmp_mtime = dump(path + "/" + i.second, sink, filter); co_yield "(";
if (tmp_mtime > result) { co_yield "name";
result = tmp_mtime; co_yield i.first;
co_yield "node";
time_t tmp_mtime;
co_yield dump(path + "/" + i.second, tmp_mtime, filter);
if (tmp_mtime > mtime) {
mtime = tmp_mtime;
} }
sink << ")"; co_yield ")";
} }
} }
else if (S_ISLNK(st.st_mode)) else if (S_ISLNK(st.st_mode)) {
sink << "type" << "symlink" << "target" << readLink(path); co_yield "type";
co_yield "symlink";
co_yield "target";
co_yield readLink(path);
}
else throw Error("file '%1%' has an unsupported type", path); else throw Error("file '%1%' has an unsupported type", path);
sink << ")"; co_yield ")";
return result;
} }
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime, PathFilter & filter)
{ {
sink << narVersionMagic1; co_yield narVersionMagic1;
return dump(path, sink, filter); co_yield dump(path, mtime, filter);
} }
void dumpPath(const Path & path, Sink & sink, PathFilter & filter) WireFormatGenerator dumpPath(Path path, PathFilter & filter)
{ {
dumpPathAndGetMtime(path, sink, filter); time_t ignored;
co_yield dumpPathAndGetMtime(path, ignored, filter);
} }
void dumpString(std::string_view s, Sink & sink) WireFormatGenerator dumpString(std::string_view s)
{ {
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; co_yield narVersionMagic1;
co_yield "(";
co_yield "type";
co_yield "regular";
co_yield "contents";
co_yield s;
co_yield ")";
} }

View file

@ -57,13 +57,13 @@ namespace nix {
* `+` denotes string concatenation. * `+` denotes string concatenation.
* ``` * ```
*/ */
void dumpPath(const Path & path, Sink & sink, WireFormatGenerator dumpPath(Path path,
PathFilter & filter = defaultPathFilter); PathFilter & filter = defaultPathFilter);
/** /**
* Same as dumpPath(), but returns the last modified date of the path. * Same as dumpPath(), but returns the last modified date of the path.
*/ */
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, WireFormatGenerator dumpPathAndGetMtime(Path path, time_t & mtime,
PathFilter & filter = defaultPathFilter); PathFilter & filter = defaultPathFilter);
/** /**
@ -71,7 +71,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
* *
* @param s Contents of the file. * @param s Contents of the file.
*/ */
void dumpString(std::string_view s, Sink & sink); WireFormatGenerator dumpString(std::string_view s);
/** /**
* \todo Fix this API, it sucks. * \todo Fix this API, it sucks.

View file

@ -370,7 +370,7 @@ HashResult hashPath(
HashType ht, const Path & path, PathFilter & filter) HashType ht, const Path & path, PathFilter & filter)
{ {
HashSink sink(ht); HashSink sink(ht);
dumpPath(path, sink, filter); sink << dumpPath(path, filter);
return sink.finish(); return sink.finish();
} }

View file

@ -101,7 +101,7 @@ struct SourcePath
void dumpPath( void dumpPath(
Sink & sink, Sink & sink,
PathFilter & filter = defaultPathFilter) const PathFilter & filter = defaultPathFilter) const
{ return nix::dumpPath(path.abs(), sink, filter); } { sink << nix::dumpPath(path.abs(), filter); }
/** /**
* Return the location of this path in the "real" filesystem, if * Return the location of this path in the "real" filesystem, if

View file

@ -671,7 +671,7 @@ static void opDump(Strings opFlags, Strings opArgs)
FdSink sink(STDOUT_FILENO); FdSink sink(STDOUT_FILENO);
std::string path = *opArgs.begin(); std::string path = *opArgs.begin();
dumpPath(path, sink); sink << dumpPath(path);
sink.flush(); sink.flush();
} }

View file

@ -30,7 +30,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
if (!namePart) namePart = baseNameOf(path); if (!namePart) namePart = baseNameOf(path);
StringSink sink; StringSink sink;
dumpPath(path, sink); sink << dumpPath(path);
auto narHash = hashString(htSHA256, sink.s); auto narHash = hashString(htSHA256, sink.s);

View file

@ -58,7 +58,7 @@ struct CmdDumpPath2 : Command
{ {
logger->pause(); logger->pause();
FdSink sink(STDOUT_FILENO); FdSink sink(STDOUT_FILENO);
dumpPath(path, sink); sink << dumpPath(path);
sink.flush(); sink.flush();
} }
}; };

View file

@ -88,7 +88,7 @@ struct CmdHashBase : Command
readFileSource(path)->drainInto(*hashSink); readFileSource(path)->drainInto(*hashSink);
break; break;
case FileIngestionMethod::Recursive: case FileIngestionMethod::Recursive:
dumpPath(path, *hashSink); *hashSink << dumpPath(path);
break; break;
} }