libstore: turn the NAR parser into a passthrough generator

this will let us turn copyNAR into a generator as well, which in turn is
necessary to turn the users of copyNAR into generators without resorting
to sinkToSource coroutines. currently this uses the SerializingTransform
in all cases, even for copyNAR where it is not necessary. should this be
a performance problem we can easily swap out the transform for one which
does not produce any bytes of its own, but that should not be necessary.

Change-Id: I7e685879318fcbb78d8b88abfddd7752360eb0ce
This commit is contained in:
eldritch horrors 2024-05-15 16:45:46 +02:00
parent 31478c810a
commit 03db4efab9
2 changed files with 26 additions and 7 deletions

View file

@ -174,9 +174,10 @@ static void skipGeneric(Source & source)
#endif #endif
static void parseContents(ParseSink & sink, Source & source, const Path & path) static WireFormatGenerator parseContents(ParseSink & sink, Source & source, const Path & path)
{ {
uint64_t size = readLongLong(source); uint64_t size = readLongLong(source);
co_yield size;
sink.preallocateContents(size); sink.preallocateContents(size);
@ -188,11 +189,13 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
auto n = buf.size(); auto n = buf.size();
if ((uint64_t)n > left) n = left; if ((uint64_t)n > left) n = left;
source(buf.data(), n); source(buf.data(), n);
co_yield std::span{buf.data(), n};
sink.receiveContents({buf.data(), n}); sink.receiveContents({buf.data(), n});
left -= n; left -= n;
} }
readPadding(size, source); readPadding(size, source);
co_yield SerializingTransform::padding(size);
} }
@ -204,12 +207,12 @@ struct CaseInsensitiveCompare
} }
}; };
static WireFormatGenerator parse(ParseSink & sink, Source & source, const Path & path)
static void parse(ParseSink & sink, Source & source, const Path & path)
{ {
std::string s; std::string s;
s = readString(source); s = readString(source);
co_yield s;
if (s != "(") throw badArchive("expected open tag"); if (s != "(") throw badArchive("expected open tag");
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
@ -220,6 +223,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
checkInterrupt(); checkInterrupt();
s = readString(source); s = readString(source);
co_yield s;
if (s == ")") { if (s == ")") {
break; break;
@ -229,6 +233,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
if (type != tpUnknown) if (type != tpUnknown)
throw badArchive("multiple type fields"); throw badArchive("multiple type fields");
std::string t = readString(source); std::string t = readString(source);
co_yield t;
if (t == "regular") { if (t == "regular") {
type = tpRegular; type = tpRegular;
@ -249,12 +254,13 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
} }
else if (s == "contents" && type == tpRegular) { else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path); co_yield parseContents(sink, source, path);
sink.closeRegularFile(); sink.closeRegularFile();
} }
else if (s == "executable" && type == tpRegular) { else if (s == "executable" && type == tpRegular) {
auto s = readString(source); auto s = readString(source);
co_yield s;
if (s != "") throw badArchive("executable marker has non-empty value"); if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable(); sink.isExecutable();
} }
@ -263,17 +269,20 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
std::string name, prevName; std::string name, prevName;
s = readString(source); s = readString(source);
co_yield s;
if (s != "(") throw badArchive("expected open tag"); if (s != "(") throw badArchive("expected open tag");
while (1) { while (1) {
checkInterrupt(); checkInterrupt();
s = readString(source); s = readString(source);
co_yield s;
if (s == ")") { if (s == ")") {
break; break;
} else if (s == "name") { } else if (s == "name") {
name = readString(source); name = readString(source);
co_yield name;
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos)
throw Error("NAR contains invalid file name '%1%'", name); throw Error("NAR contains invalid file name '%1%'", name);
if (name <= prevName) if (name <= prevName)
@ -290,7 +299,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
} }
} else if (s == "node") { } else if (s == "node") {
if (name.empty()) throw badArchive("entry name missing"); if (name.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name); co_yield parse(sink, source, path + "/" + name);
} else } else
throw badArchive("unknown field " + s); throw badArchive("unknown field " + s);
} }
@ -298,6 +307,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
else if (s == "target" && type == tpSymlink) { else if (s == "target" && type == tpSymlink) {
std::string target = readString(source); std::string target = readString(source);
co_yield target;
sink.createSymlink(path, target); sink.createSymlink(path, target);
} }
@ -307,20 +317,28 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
} }
void parseDump(ParseSink & sink, Source & source) WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source)
{ {
std::string version; std::string version;
try { try {
version = readString(source, narVersionMagic1.size()); version = readString(source, narVersionMagic1.size());
co_yield version;
} catch (SerialisationError & e) { } catch (SerialisationError & e) {
/* This generally means the integer at the start couldn't be /* This generally means the integer at the start couldn't be
decoded. Ignore and throw the exception below. */ decoded. Ignore and throw the exception below. */
} }
if (version != narVersionMagic1) if (version != narVersionMagic1)
throw badArchive("input doesn't look like a Nix archive"); throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, ""); co_yield parse(sink, source, "");
} }
void parseDump(ParseSink & sink, Source & source)
{
auto parser = parseAndCopyDump(sink, source);
while (parser.next()) {
// ignore the actual item
}
}
struct RestoreSink : ParseSink struct RestoreSink : ParseSink
{ {

View file

@ -116,6 +116,7 @@ struct RetrieveRegularNARSink : ParseSink
} }
}; };
WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source);
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink & sink, Source & source);
void restorePath(const Path & path, Source & source); void restorePath(const Path & path, Source & source);