nrabulinski/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

libstore: have path dump return a generator

Browse source 
Change-Id: Ic4cf5562504aa29130304469936f958c0426e5ef
This commit is contained in:
eldritch horrors 2024-03-23 00:47:16 +01:00
parent 05c04089ee
commit 6f30245e0a
20 changed files with 122 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/libfetchers/path.cc
View file

@ -130,10 +130,8 @@ struct PathInputScheme : InputScheme
time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
// FIXME: try to substitute storePath.
auto src = sinkToSource([&](Sink & sink) {
mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
});
storePath = store->addToStoreFromDump(*src, "source");
auto src = WireSource{dumpPathAndGetMtime(absPath, mtime, defaultPathFilter)};
storePath = store->addToStoreFromDump(src, "source");
}
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));

2
src/libfetchers/tarball.cc
View file

@ -71,7 +71,7 @@ DownloadFileResult downloadFile(
storePath = std::move(cached->storePath);
} else {
StringSink sink;
dumpString(res.data, sink);
sink << dumpString(res.data);
auto hash = hashString(htSHA256, res.data);
ValidPathInfo info {
*store,

11
src/libstore/binary-cache-store.cc
View file

@ -5,6 +5,7 @@
#include "fs-accessor.hh"
#include "globals.hh"
#include "nar-info.hh"
#include "serialise.hh"
#include "sync.hh"
#include "remote-fs-accessor.hh"
#include "nar-info-disk-cache.hh"
@ -413,16 +414,14 @@ StorePath BinaryCacheStore::addToStore(
HashSink sink { hashAlgo };
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
sink << dumpPath(srcPath, filter);
} else {
readFile(srcPath, sink);
}
auto h = sink.finish().first;
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink, filter);
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
auto source = WireSource{dumpPath(srcPath, filter)};
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
name,
@ -455,7 +454,7 @@ StorePath BinaryCacheStore::addTextToStore(
return path;
StringSink sink;
dumpString(s, sink);
sink << dumpString(s);
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {

4
src/libstore/build/local-derivation-goal.cc
View file

@ -2393,7 +2393,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
/* FIXME: Is this actually streaming? */
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink(rewrites, nextSink);
dumpPath(actualPath, rsink);
rsink << dumpPath(actualPath);
rsink.flush();
});
Path tmpPath = actualPath + ".tmp";
@ -2459,7 +2459,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink);
caSink << dumpPath(actualPath);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);

2
src/libstore/daemon.cc
View file

@ -867,7 +867,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
logger->stopWork();
dumpPath(store->toRealPath(path), to);
to << dumpPath(store->toRealPath(path));
break;
}

2
src/libstore/local-fs-store.cc
View file

@ -82,7 +82,7 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
{
if (!isValidPath(path))
throw Error("path '%s' is not valid", printStorePath(path));
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
sink << dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()));
}
const std::string LocalFSStore::drvsLogDir = "drvs";

8
src/libstore/local-store.cc
View file

@ -1395,7 +1395,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto narHash = std::pair { hash, size };
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
HashSink narSink { htSHA256 };
dumpPath(realPath, narSink);
narSink << dumpPath(realPath);
narHash = narSink.finish();
}
@ -1450,7 +1450,7 @@ StorePath LocalStore::addTextToStore(
canonicalisePathMetaData(realPath, {});
StringSink sink;
dumpString(s, sink);
sink << dumpString(s);
auto narHash = hashString(htSHA256, sink.s);
optimisePath(realPath, repair);
@ -1590,7 +1590,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
auto hashSink = HashSink(info->narHash.type);
dumpPath(Store::toRealPath(i), hashSink);
hashSink << dumpPath(Store::toRealPath(i));
auto current = hashSink.finish();
if (info->narHash != nullHash && info->narHash != current.first) {
@ -1891,7 +1891,7 @@ ContentAddress LocalStore::hashCAPath(
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
caSink << dumpPath(path);
break;
case FileIngestionMethod::Flat:
readFile(path, caSink);

2
src/libstore/path-references.cc
View file

@ -65,7 +65,7 @@ StorePathSet scanForReferences(
TeeSink sink { refsSink, toTee };
/* Look for the hashes in the NAR dump of the path. */
dumpPath(path, sink);
sink << dumpPath(path);
return refsSink.getResultPaths();
}

2
src/libstore/remote-store.cc
View file

@ -473,7 +473,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
dump.drainInto(conn->to);
} else {
std::string contents = dump.drain();
dumpString(contents, conn->to);
conn->to << dumpString(contents);
}
}
conn.processStderr();

15
src/libstore/store-api.cc
View file

@ -3,6 +3,7 @@
#include "globals.hh"
#include "derivations.hh"
#include "store-api.hh"
#include "serialise.hh"
#include "util.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
@ -272,7 +273,7 @@ StorePath Store::addToStore(
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
sink << dumpPath(srcPath, filter);
else
readFile(srcPath, sink);
});
@ -421,13 +422,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
/* Functionally, this means that fileSource will yield the content of
srcPath. The fact that we use scratchpadSink as a temporary buffer here
is an implementation detail. */
auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink);
});
auto fileSource = WireSource{dumpPath(srcPath)};
/* tapped provides the same data as fileSource, but we also write all the
information to narSink. */
TeeSource tapped { *fileSource, narSink };
TeeSource tapped { fileSource, narSink };
ParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat
@ -462,10 +461,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
info.narSize = narSize;
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink);
});
addToStore(info, *source);
auto source = WireSource{dumpPath(srcPath)};
addToStore(info, source);
}
return info;

85
src/libutil/archive.cc
View file

@ -12,6 +12,7 @@
#include <fcntl.h>
#include "archive.hh"
#include "serialise.hh"
#include "util.hh"
#include "config.hh"
#include "signals.hh"
@ -39,10 +40,10 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, off_t size,
Sink & sink)
static WireFormatGenerator dumpContents(const Path & path, off_t size)
{
sink << "contents" << size;
co_yield "contents";
co_yield size;
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
if (!fd) throw SysError("opening file '%1%'", path);
@ -54,31 +55,35 @@ static void dumpContents(const Path & path, off_t size,
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink({buf.data(), n});
co_yield std::span{buf.data(), n};
}
writePadding(size, sink);
co_yield SerializingTransform::padding(size);
}
static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
static WireFormatGenerator dump(const Path & path, time_t & mtime, PathFilter & filter)
{
checkInterrupt();
auto st = lstat(path);
time_t result = st.st_mtime;
mtime = st.st_mtime;
sink << "(";
co_yield "(";
if (S_ISREG(st.st_mode)) {
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
dumpContents(path, st.st_size, sink);
co_yield "type";
co_yield "regular";
if (st.st_mode & S_IXUSR) {
co_yield "executable";
co_yield "";
}
co_yield dumpContents(path, st.st_size);
}
else if (S_ISDIR(st.st_mode)) {
sink << "type" << "directory";
co_yield "type";
co_yield "directory";
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
@ -100,41 +105,55 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
for (auto & i : unhacked)
if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node";
auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
if (tmp_mtime > result) {
result = tmp_mtime;
co_yield "entry";
co_yield "(";
co_yield "name";
co_yield i.first;
co_yield "node";
time_t tmp_mtime;
co_yield dump(path + "/" + i.second, tmp_mtime, filter);
if (tmp_mtime > mtime) {
mtime = tmp_mtime;
}
sink << ")";
co_yield ")";
}
}
else if (S_ISLNK(st.st_mode))
sink << "type" << "symlink" << "target" << readLink(path);
else if (S_ISLNK(st.st_mode)) {
co_yield "type";
co_yield "symlink";
co_yield "target";
co_yield readLink(path);
}
else throw Error("file '%1%' has an unsupported type", path);
sink << ")";
return result;
co_yield ")";
}
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime, PathFilter & filter)
{
sink << narVersionMagic1;
return dump(path, sink, filter);
co_yield narVersionMagic1;
co_yield dump(path, mtime, filter);
}
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
WireFormatGenerator dumpPath(const Path & path, PathFilter & filter)
{
dumpPathAndGetMtime(path, sink, filter);
time_t ignored;
co_yield dumpPathAndGetMtime(path, ignored, filter);
}
void dumpString(std::string_view s, Sink & sink)
WireFormatGenerator dumpString(std::string_view s)
{
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
co_yield narVersionMagic1;
co_yield "(";
co_yield "type";
co_yield "regular";
co_yield "contents";
co_yield s;
co_yield ")";
}
@ -391,10 +410,8 @@ void copyNAR(Source & source, Sink & sink)
void copyPath(const Path & from, const Path & to)
{
auto source = sinkToSource([&](Sink & sink) {
dumpPath(from, sink);
});
restorePath(to, *source);
auto source = WireSource{dumpPath(from)};
restorePath(to, source);
}

6
src/libutil/archive.hh
View file

@ -56,13 +56,13 @@ namespace nix {
* `+` denotes string concatenation.
* ```
*/
void dumpPath(const Path & path, Sink & sink,
WireFormatGenerator dumpPath(const Path & path,
PathFilter & filter = defaultPathFilter);
/**
* Same as dumpPath(), but returns the last modified date of the path.
*/
time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
WireFormatGenerator dumpPathAndGetMtime(const Path & path, time_t & mtime,
PathFilter & filter = defaultPathFilter);
/**
@ -70,7 +70,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
*
* @param s Contents of the file.
*/
void dumpString(std::string_view s, Sink & sink);
WireFormatGenerator dumpString(std::string_view s);
/**
* \todo Fix this API, it sucks.

2
src/libutil/hash.cc
View file

@ -371,7 +371,7 @@ HashResult hashPath(
HashType ht, const Path & path, PathFilter & filter)
{
HashSink sink(ht);
dumpPath(path, sink, filter);
sink << dumpPath(path, filter);
return sink.finish();
}

37
src/libutil/serialise.hh
View file

@ -385,6 +385,30 @@ inline void drainGenerator(WireFormatGenerator g, std::derived_from<Sink> auto &
}
}
struct WireSource : Source
{
WireSource(WireFormatGenerator && g) : g(std::move(g)) {}
virtual size_t read(char * data, size_t len)
{
while (!buf.size() && g) {
buf = g();
}
if (!buf.size()) {
throw EndOfFile("coroutine has finished");
}
len = std::min(len, buf.size());
memcpy(data, buf.data(), len);
buf = buf.subspan(len);
return len;
}
private:
WireFormatGenerator g;
std::span<const char> buf{};
};
struct SerializingTransform
{
std::array<char, 8> buf;
@ -407,6 +431,19 @@ struct SerializingTransform
return {buf.begin(), 8};
}
static std::span<const char> padding(size_t unpadded)
{
return std::span("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0);
}
// opt in to generator chaining. without this co_yielding
// another generator of any type will cause a type error.
template<typename TF>
auto operator()(Generator<std::span<const char>, TF> && g)
{
return std::move(g);
}
// only choose this for *exactly* char spans, do not allow implicit
// conversions. this would cause ambiguities with strings literals,
// and resolving those with more string-like overloads needs a lot.

2
src/libutil/source-path.hh
View file

@ -99,7 +99,7 @@ struct SourcePath
void dumpPath(
Sink & sink,
PathFilter & filter = defaultPathFilter) const
{ return nix::dumpPath(path.abs(), sink, filter); }
{ sink << nix::dumpPath(path.abs(), filter); }
/**
* Return the location of this path in the "real" filesystem, if

2
src/nix-store/nix-store.cc
View file

@ -672,7 +672,7 @@ static void opDump(Strings opFlags, Strings opArgs)
FdSink sink(STDOUT_FILENO);
std::string path = *opArgs.begin();
dumpPath(path, sink);
sink << dumpPath(path);
sink.flush();
}

2
src/nix/add-to-store.cc
View file

@ -30,7 +30,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
if (!namePart) namePart = baseNameOf(path);
StringSink sink;
dumpPath(path, sink);
sink << dumpPath(path);
auto narHash = hashString(htSHA256, sink.s);

2
src/nix/dump-path.cc
View file

@ -56,7 +56,7 @@ struct CmdDumpPath2 : Command
void run() override
{
FdSink sink(STDOUT_FILENO);
dumpPath(path, sink);
sink << dumpPath(path);
sink.flush();
}
};

2
src/nix/hash.cc
View file

@ -88,7 +88,7 @@ struct CmdHashBase : Command
readFile(path, *hashSink);
break;
case FileIngestionMethod::Recursive:
dumpPath(path, *hashSink);
*hashSink << dumpPath(path);
break;
}

2
src/nix/profile.cc
View file

@ -215,7 +215,7 @@ struct ProfileManifest
/* Add the symlink tree to the store. */
StringSink sink;
dumpPath(tempDir, sink);
sink << dumpPath(tempDir);
auto narHash = hashString(htSHA256, sink.s);