From 0a9da00a10fa27a3e3b98439cb0a7d5e79135b58 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 17:30:42 +0200 Subject: [PATCH] NarAccessor: Run in constant memory --- src/libstore/binary-cache-store.cc | 16 +++++------ src/libstore/daemon.cc | 9 +++--- src/libstore/export-import.cc | 6 ++-- src/libstore/nar-accessor.cc | 46 +++++++++++++++++++----------- src/libstore/nar-accessor.hh | 4 +++ src/libutil/archive.hh | 3 +- src/libutil/serialise.hh | 10 +++---- 7 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 166041b6c..5851722d7 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -155,13 +155,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto now1 = std::chrono::steady_clock::now(); + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink(htSHA256); - + std::shared_ptr narAccessor; { FdSink fileSink(fdTemp.get()); TeeSink teeSink(fileSink, fileHashSink); auto compressionSink = makeCompressionSink(compression, teeSink); - copyNAR(narSource, *compressionSink); + TeeSource teeSource(narSource, *compressionSink); + narAccessor = makeNarAccessor(teeSource); compressionSink->finish(); } @@ -200,10 +204,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource #if 0 auto accessor_ = std::dynamic_pointer_cast(accessor); - auto narAccessor = makeNarAccessor(nar); - if (accessor_) accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); + #endif /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -216,15 +219,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref(narAccessor), "", true); } } upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } - #endif - #if 0 /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ @@ -281,7 +282,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource threadPool.process(); } } - #endif /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f05f4739d..536f4e738 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -391,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref store, } HashType hashAlgo = parseHashType(s); - TeeSource savedNAR(from); + StringSink savedNAR; + TeeSource savedNARSource(from, savedNAR); RetrieveRegularNARSink savedRegular; if (method == FileIngestionMethod::Recursive) { @@ -399,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref store, a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); + parseDump(sink, savedNARSource); } else parseDump(savedRegular, from); @@ -407,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref store, if (!savedRegular.regular) throw Error("regular file expected"); auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, baseName, method, hashAlgo); @@ -733,7 +734,7 @@ static void performOp(TunnelLogger * logger, ref store, else { TeeParseSink tee(from); parseDump(tee, tee.source); - saved = std::move(*tee.source.data); + saved = std::move(*tee.saved.s); source = std::make_unique(saved); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cfed8ccd8..ae31cbcb0 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -77,15 +77,15 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *tee.saved.s); + info.narSize = tee.saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; + auto source = StringSource { *tee.saved.s }; addToStore(info, source, NoRepair, checkSigs, accessor); res.push_back(info.path); diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ca663d837..d884a131e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -18,7 +18,7 @@ struct NarMember /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start = 0, size = 0; + uint64_t start = 0, size = 0; std::string target; @@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, StringSource + struct NarIndexer : ParseSink, Source { NarAccessor & acc; + Source & source; std::stack parents; - std::string currentStart; bool isExec = false; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) + uint64_t pos = 0; + + NarIndexer(NarAccessor & acc, Source & source) + : acc(acc), source(source) { } void createMember(const Path & path, NarMember member) { @@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor void preallocateContents(unsigned long long size) override { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits::max()); - parents.top()->size = (size_t)size; + assert(size <= std::numeric_limits::max()); + parents.top()->size = (uint64_t) size; parents.top()->start = pos; } void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } + { } void createSymlink(const Path & path, const string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + + size_t read(unsigned char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } }; NarAccessor(ref nar) : nar(nar) { - NarIndexer indexer(*this, *nar); + StringSource source(*nar); + NarIndexer indexer(*this, source); + parseDump(indexer, indexer); + } + + NarAccessor(Source & source) + { + NarIndexer indexer(*this, source); parseDump(indexer, indexer); } @@ -219,6 +228,11 @@ ref makeNarAccessor(ref nar) return make_ref(nar); } +ref makeNarAccessor(Source & source) +{ + return make_ref(source); +} + ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 2871199de..8af1272f6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -6,10 +6,14 @@ namespace nix { +struct Source; + /* Return an object that provides access to the contents of a NAR file. */ ref makeNarAccessor(ref nar); +ref makeNarAccessor(Source & source); + /* Create a NAR accessor from a NAR listing (in the format produced by listNar()). The callback getNarBytes(offset, length) is used by the readFile() method of the accessor to get the contents of files diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 795e9ce02..302b1bb18 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -65,9 +65,10 @@ struct ParseSink struct TeeParseSink : ParseSink { + StringSink saved; TeeSource source; - TeeParseSink(Source & source) : source(source) { } + TeeParseSink(Source & source) : source(source, saved) { } }; void parseDump(ParseSink & sink, Source & source); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index bd651fb7d..84a4eb001 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,17 +179,17 @@ struct TeeSink : Sink }; -/* Adapter class of a Source that saves all data read to `s'. */ +/* Adapter class of a Source that saves all data read to a sink. */ struct TeeSource : Source { Source & orig; - ref data; - TeeSource(Source & orig) - : orig(orig), data(make_ref()) { } + Sink & sink; + TeeSource(Source & orig, Sink & sink) + : orig(orig), sink(sink) { } size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - this->data->append((const char *) data, n); + sink(data, len); return n; } };