From 542ae5c8f804b704101f9d27bd8b2aa62ded899c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Oct 2016 16:50:28 +0200 Subject: [PATCH] BinaryCacheStore: Optionally write a NAR listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The store parameter "write-nar-listing=1" will cause BinaryCacheStore to write a file ‘.ls.xz’ for each ‘.narinfo’ added to the binary cache. This file contains an XZ-compressed JSON file describing the contents of the NAR, excluding the contents of regular files. E.g. { "version": 1, "root": { "type": "directory", "entries": { "lib": { "type": "directory", "entries": { "Mcrt1.o": { "type": "regular", "size": 1288 }, "Scrt1.o": { "type": "regular", "size": 3920 }, } } } ... } } (The actual file has no indentation.) This is intended to speed up the NixOS channels programs index generator [1], since fetching gazillions of large NARs from cache.nixos.org is currently a bottleneck for updating the regular (non-small) channel. [1] https://github.com/NixOS/nixos-channel-scripts/blob/master/generate-programs-index.cc --- src/libstore/binary-cache-store.cc | 73 ++++++++++++++++++++++++++---- src/libstore/binary-cache-store.hh | 8 +++- src/libstore/download.cc | 2 +- src/libstore/export-import.cc | 2 +- src/libstore/local-store.cc | 6 +-- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 +- 10 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 01d1a7e24..7692da742 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -9,6 +9,7 @@ #include "worker-protocol.hh" #include "nar-accessor.hh" #include "nar-info-disk-cache.hh" +#include "json.hh" #include @@ -19,6 +20,7 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) , compression(get(params, "compression", "xz")) + , writeNARListing(get(params, "write-nar-listing", "0") == "1") { auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") @@ -79,7 +81,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar, +void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref & nar, bool repair, bool dontCheckSigs) { if (!repair && isValidPath(info.path)) return; @@ -97,20 +99,73 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string auto narInfoFile = narInfoFileFor(info.path); - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + /* Optionally write a JSON file containing a listing of the + contents of the NAR. */ + if (writeNARListing) { + std::ostringstream jsonOut; + + { + JSONObject jsonRoot(jsonOut); + jsonRoot.attr("version", 1); + + auto accessor = makeNarAccessor(nar); + + std::function recurse; + + recurse = [&](const Path & path, JSONPlaceholder & res) { + auto st = accessor->stat(path); + + auto obj = res.object(); + + switch (st.type) { + case FSAccessor::Type::tRegular: + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) + obj.attr("executable", true); + break; + case FSAccessor::Type::tDirectory: + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto & name : accessor->readDirectory(path)) { + auto res3 = res2.placeholder(name); + recurse(path + "/" + name, res3); + } + } + break; + case FSAccessor::Type::tSymlink: + obj.attr("type", "symlink"); + obj.attr("target", accessor->readLink(path)); + break; + default: + abort(); + } + }; + + { + auto res = jsonRoot.placeholder("root"); + recurse("", res); + } + } + + upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str())); + } + /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, nar); + auto narCompressed = compress(compression, *nar); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); @@ -118,7 +173,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ @@ -132,7 +187,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); + stats.narWriteBytes += nar->size(); stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; @@ -231,7 +286,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, ValidPathInfo info; info.path = makeFixedOutputPath(recursive, h, name); - addToStore(info, *sink.s, repair); + addToStore(info, sink.s, repair); return info.path; } @@ -246,7 +301,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, *sink.s, repair); + addToStore(info, sink.s, repair); } return info.path; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 574696cf3..333cf0856 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -19,12 +19,16 @@ private: std::string compression; + bool writeNARListing; + protected: BinaryCacheStore(const Params & params); [[noreturn]] void notImpl(); +public: + virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, const std::string & data) = 0; @@ -37,6 +41,8 @@ protected: std::shared_ptr getFile(const std::string & path); +protected: + bool wantMassQuery_ = false; int priority = 50; @@ -86,7 +92,7 @@ public: bool wantMassQuery() override { return wantMassQuery_; } - void addToStore(const ValidPathInfo & info, const std::string & nar, + void addToStore(const ValidPathInfo & info, const ref & nar, bool repair = false, bool dontCheckSigs = false) override; Path addToStore(const string & name, const Path & srcPath, diff --git a/src/libstore/download.cc b/src/libstore/download.cc index c01ba63ef..954044c23 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -558,7 +558,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); - store->addToStore(info, *sink.s, false, true); + store->addToStore(info, sink.s, false, true); storePath = info.path; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b7f43acf1..883b87217 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -117,7 +117,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr accessor, if (readInt(source) == 1) readString(source); - addToStore(info, *tee.data, false, dontCheckSigs); + addToStore(info, tee.data, false, dontCheckSigs); if (accessor) addPathToAccessor(ref(accessor), info.path, tee.data); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ed5747f31..2c2475afc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -909,10 +909,10 @@ void LocalStore::invalidatePath(State & state, const Path & path) } -void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar, +void LocalStore::addToStore(const ValidPathInfo & info, const ref & nar, bool repair, bool dontCheckSigs) { - Hash h = hashString(htSHA256, nar); + Hash h = hashString(htSHA256, *nar); if (h != info.narHash) throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % info.path % info.narHash.to_string() % h.to_string()); @@ -939,7 +939,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar, deletePath(realPath); - StringSource source(nar); + StringSource source(*nar); restorePath(realPath, source); canonicalisePathMetaData(realPath, -1); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 24188130d..86f03526d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -125,7 +125,7 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const std::string & nar, + void addToStore(const ValidPathInfo & info, const ref & nar, bool repair, bool dontCheckSigs) override; Path addToStore(const string & name, const Path & srcPath, diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 38af145f9..de51716db 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) } -void RemoteStore::addToStore(const ValidPathInfo & info, const std::string & nar, +void RemoteStore::addToStore(const ValidPathInfo & info, const ref & nar, bool repair, bool dontCheckSigs) { throw Error("RemoteStore::addToStore() not implemented"); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9879337d6..5932fda95 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -53,7 +53,7 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const std::string & nar, + void addToStore(const ValidPathInfo & info, const ref & nar, bool repair, bool dontCheckSigs) override; Path addToStore(const string & name, const Path & srcPath, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b6ac1c3b9..f365406cb 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -456,7 +456,7 @@ void copyStorePath(ref srcStore, ref dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); - dstStore->addToStore(*info, *sink.s, repair); + dstStore->addToStore(*info, sink.s, repair); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ce1583b0c..d6e181196 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -365,7 +365,7 @@ public: virtual bool wantMassQuery() { return false; } /* Import a path into the store. */ - virtual void addToStore(const ValidPathInfo & info, const std::string & nar, + virtual void addToStore(const ValidPathInfo & info, const ref & nar, bool repair = false, bool dontCheckSigs = false) = 0; /* Copy the contents of a path to the store and register the