From 263187a2ec0a2ddd16cfd8c54c55e3455c9cd987 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2016 14:48:16 +0100 Subject: [PATCH] Move BinaryCacheStore / LocalBinaryCacheStore from Hydra So you can now do: $ NIX_REMOTE=file:///tmp/binary-cache nix-store -qR /nix/store/... --- src/libstore/binary-cache-store.cc | 292 +++++++++++++++++++++++ src/libstore/binary-cache-store.hh | 170 +++++++++++++ src/libstore/local-binary-cache-store.cc | 44 ++++ src/libstore/local-binary-cache-store.hh | 31 +++ src/libstore/store-api.cc | 18 +- src/libstore/store-api.hh | 24 +- src/libutil/lru-cache.hh | 84 +++++++ 7 files changed, 658 insertions(+), 5 deletions(-) create mode 100644 src/libstore/binary-cache-store.cc create mode 100644 src/libstore/binary-cache-store.hh create mode 100644 src/libstore/local-binary-cache-store.cc create mode 100644 src/libstore/local-binary-cache-store.hh create mode 100644 src/libutil/lru-cache.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc new file mode 100644 index 000000000..6bea0e6c9 --- /dev/null +++ b/src/libstore/binary-cache-store.cc @@ -0,0 +1,292 @@ +#include "binary-cache-store.hh" +#include "sync.hh" + +#include "archive.hh" +#include "compression.hh" +#include "derivations.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "worker-protocol.hh" + +#include + +namespace nix { + +BinaryCacheStore::BinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile) + : localStore(localStore) +{ + if (secretKeyFile != "") + secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + + if (publicKeyFile != "") { + publicKeys = std::unique_ptr(new PublicKeys); + auto key = PublicKey(readFile(publicKeyFile)); + publicKeys->emplace(key.name, key); + } +} + +void BinaryCacheStore::init() +{ + std::string cacheInfoFile = "nix-cache-info"; + if (!fileExists(cacheInfoFile)) + upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n"); +} + +const BinaryCacheStore::Stats & BinaryCacheStore::getStats() +{ + return stats; +} + +Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +{ + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; +} + +void BinaryCacheStore::addToCache(const ValidPathInfo & info, + const string & nar) +{ + auto narInfoFile = narInfoFileFor(info.path); + if (fileExists(narInfoFile)) return; + + auto narInfo = make_ref(info); + + narInfo->narSize = nar.size(); + narInfo->narHash = hashString(htSHA256, nar); + + if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash) + throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + + /* Compress the NAR. */ + narInfo->compression = "xz"; + auto now1 = std::chrono::steady_clock::now(); + string narXz = compressXZ(nar); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, narXz); + narInfo->fileSize = narXz.size(); + + auto duration = std::chrono::duration_cast(now2 - now1).count(); + printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") + % narInfo->path % narInfo->narSize + % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % duration); + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + if (!fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, narXz); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar.size(); + stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*secretKey); + + upsertFile(narInfoFile, narInfo->to_string()); + + { + auto state_(state.lock()); + state_->narInfoCache.upsert(narInfo->path, narInfo); + stats.narInfoCacheSize = state_->narInfoCache.size(); + } + + stats.narInfoWrite++; +} + +NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) +{ + { + auto state_(state.lock()); + auto res = state_->narInfoCache.get(storePath); + if (res) { + stats.narInfoReadAverted++; + return **res; + } + } + + auto narInfoFile = narInfoFileFor(storePath); + auto narInfo = make_ref(getFile(narInfoFile), narInfoFile); + assert(narInfo->path == storePath); + + stats.narInfoRead++; + + if (publicKeys) { + if (!narInfo->checkSignature(*publicKeys)) + throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile); + } + + { + auto state_(state.lock()); + state_->narInfoCache.upsert(storePath, narInfo); + stats.narInfoCacheSize = state_->narInfoCache.size(); + } + + return *narInfo; +} + +bool BinaryCacheStore::isValidPath(const Path & storePath) +{ + return fileExists(narInfoFileFor(storePath)); +} + +void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) +{ + assert(!sign); + + auto res = readNarInfo(storePath); + + auto nar = getFile(res.url); + + stats.narRead++; + stats.narReadCompressedBytes += nar.size(); + + /* Decompress the NAR. FIXME: would be nice to have the remote + side do this. */ + if (res.compression == "none") + ; + else if (res.compression == "xz") + nar = decompressXZ(nar); + else + throw Error(format("unknown NAR compression type ‘%1%’") % nar); + + stats.narReadBytes += nar.size(); + + printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size()); + + assert(nar.size() % 8 == 0); + + sink((unsigned char *) nar.c_str(), nar.size()); + + // FIXME: check integrity of NAR. + + sink << exportMagic << storePath << res.references << res.deriver << 0; +} + +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) +{ + assert(!requireSignature); + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); + res.push_back(importPath(source)); + } + return res; +} + +struct TeeSource : Source +{ + Source & readSource; + std::string data; + TeeSource(Source & readSource) : readSource(readSource) + { + } + size_t read(unsigned char * data, size_t len) + { + size_t n = readSource.read(data, len); + this->data.append((char *) data, n); + return n; + } +}; + +struct NopSink : ParseSink +{ +}; + +Path BinaryCacheStore::importPath(Source & source) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, tee.data); + + return info.path; +} + +ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) +{ + return ValidPathInfo(readNarInfo(storePath)); +} + +void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + PathSet left; + + if (!localStore) return; + + for (auto & storePath : paths) { + if (!localStore->isValidPath(storePath)) { + left.insert(storePath); + continue; + } + ValidPathInfo info = localStore->queryPathInfo(storePath); + SubstitutablePathInfo sub; + sub.references = info.references; + sub.downloadSize = 0; + sub.narSize = info.narSize; + infos.emplace(storePath, sub); + } + + if (settings.useSubstitutes) + localStore->querySubstitutablePathInfos(left, infos); +} + +void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & storePath : paths) { + assert(!isDerivation(storePath)); + + if (isValidPath(storePath)) continue; + + if (!localStore) + throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath); + + localStore->addTempRoot(storePath); + + if (!localStore->isValidPath(storePath)) + localStore->ensurePath(storePath); + + ValidPathInfo info = localStore->queryPathInfo(storePath); + + for (auto & ref : info.references) + if (ref != storePath) + ensurePath(ref); + + StringSink sink; + dumpPath(storePath, sink); + + addToCache(info, sink.s); + } +} + +void BinaryCacheStore::ensurePath(const Path & path) +{ + buildPaths({path}); +} + +} diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh new file mode 100644 index 000000000..d02f46dea --- /dev/null +++ b/src/libstore/binary-cache-store.hh @@ -0,0 +1,170 @@ +#pragma once + +#include "crypto.hh" +#include "store-api.hh" + +#include "lru-cache.hh" +#include "sync.hh" +#include "pool.hh" + +#include + +namespace nix { + +struct NarInfo; + +class BinaryCacheStore : public Store +{ +private: + + std::unique_ptr secretKey; + std::unique_ptr publicKeys; + + std::shared_ptr localStore; + + struct State + { + LRUCache> narInfoCache{32 * 1024}; + }; + + Sync state; + +protected: + + BinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile); + + virtual bool fileExists(const std::string & path) = 0; + + virtual void upsertFile(const std::string & path, const std::string & data) = 0; + + virtual std::string getFile(const std::string & path) = 0; + +public: + + virtual void init(); + + struct Stats + { + std::atomic narInfoRead{0}; + std::atomic narInfoReadAverted{0}; + std::atomic narInfoWrite{0}; + std::atomic narInfoCacheSize{0}; + std::atomic narRead{0}; + std::atomic narReadBytes{0}; + std::atomic narReadCompressedBytes{0}; + std::atomic narWrite{0}; + std::atomic narWriteAverted{0}; + std::atomic narWriteBytes{0}; + std::atomic narWriteCompressedBytes{0}; + std::atomic narWriteCompressionTimeMs{0}; + }; + + const Stats & getStats(); + +private: + + Stats stats; + + std::string narInfoFileFor(const Path & storePath); + + void addToCache(const ValidPathInfo & info, const string & nar); + +protected: + + NarInfo readNarInfo(const Path & storePath); + +public: + + bool isValidPath(const Path & path) override; + + PathSet queryValidPaths(const PathSet & paths) override + { abort(); } + + PathSet queryAllValidPaths() override + { abort(); } + + ValidPathInfo queryPathInfo(const Path & path) override; + + Hash queryPathHash(const Path & path) override + { abort(); } + + void queryReferrers(const Path & path, + PathSet & referrers) override + { abort(); } + + Path queryDeriver(const Path & path) override + { abort(); } + + PathSet queryValidDerivers(const Path & path) override + { abort(); } + + PathSet queryDerivationOutputs(const Path & path) override + { abort(); } + + StringSet queryDerivationOutputNames(const Path & path) override + { abort(); } + + Path queryPathFromHashPart(const string & hashPart) override + { abort(); } + + PathSet querySubstitutablePaths(const PathSet & paths) override + { abort(); } + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false) override + { abort(); } + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false) override + { abort(); } + + void exportPath(const Path & path, bool sign, + Sink & sink) override; + + Paths importPaths(bool requireSignature, Source & source) override; + + Path importPath(Source & source); + + void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { abort(); } + + void ensurePath(const Path & path) override; + + void addTempRoot(const Path & path) override + { abort(); } + + void addIndirectRoot(const Path & path) override + { abort(); } + + void syncWithGC() override + { } + + Roots findRoots() override + { abort(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { abort(); } + + PathSet queryFailedPaths() override + { return PathSet(); } + + void clearFailedPaths(const PathSet & paths) override + { } + + void optimiseStore() override + { } + + bool verifyStore(bool checkContents, bool repair) override + { return true; } + +}; + +} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000..5714688e0 --- /dev/null +++ b/src/libstore/local-binary-cache-store.cc @@ -0,0 +1,44 @@ +#include "local-binary-cache-store.hh" + +namespace nix { + +LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir) + : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) + , binaryCacheDir(binaryCacheDir) +{ +} + +void LocalBinaryCacheStore::init() +{ + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); +} + +static void atomicWrite(const Path & path, const std::string & s) +{ + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string & path) +{ + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +{ + atomicWrite(binaryCacheDir + "/" + path, data); +} + +std::string LocalBinaryCacheStore::getFile(const std::string & path) +{ + return readFile(binaryCacheDir + "/" + path); +} + +} diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 000000000..0303ebe73 --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "binary-cache-store.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); + + void init() override; + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 82872cc33..f9c876770 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -313,18 +313,24 @@ void Store::exportPaths(const Paths & paths, #include "local-store.hh" -#include "serialise.hh" #include "remote-store.hh" +#include "local-binary-cache-store.hh" namespace nix { -ref openStore(bool reserveSpace) +ref openStoreAt(const std::string & uri, bool reserveSpace) { + if (std::string(uri, 0, 7) == "file://") { + return make_ref(std::shared_ptr(0), + "", "", // FIXME: allow the signing key to be set + std::string(uri, 7)); + } + enum { mDaemon, mLocal, mAuto } mode; - mode = getEnv("NIX_REMOTE") == "daemon" ? mDaemon : mAuto; + mode = uri == "daemon" ? mDaemon : mAuto; if (mode == mAuto) { if (LocalStore::haveWriteAccess()) @@ -341,4 +347,10 @@ ref openStore(bool reserveSpace) } +ref openStore(bool reserveSpace) +{ + return openStoreAt(getEnv("NIX_REMOTE"), reserveSpace); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6f50e3c55..5fe42c973 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -419,8 +419,28 @@ Path computeStorePathForText(const string & name, const string & s, void removeTempRoots(); -/* Factory method: open the Nix database, either through the local or - remote implementation. */ +/* Return a Store object to access the Nix store denoted by + ‘uri’ (slight misnomer...). Supported values are: + + * ‘direct’: The Nix store in /nix/store and database in + /nix/var/nix/db, accessed directly. + + * ‘daemon’: The Nix store accessed via a Unix domain socket + connection to nix-daemon. + + * ‘file://’: A binary cache stored in . + + If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on + whether the user has write access to the local Nix store/database. + + The Boolean ‘reserveSpace’ denotes whether some disk space should + be reserved to enable future garbage collector runs. It should be + set to true *unless* you're going to collect garbage. +*/ +ref openStoreAt(const std::string & uri, bool reserveSpace = true); + + +/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */ ref openStore(bool reserveSpace = true); diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh new file mode 100644 index 000000000..3c6569cf4 --- /dev/null +++ b/src/libutil/lru-cache.hh @@ -0,0 +1,84 @@ +#pragma once + +#include +#include + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template +class LRUCache +{ +private: + + size_t maxSize; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map>; + using LRU = std::list; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t maxSize) : maxSize(maxSize) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + erase(key); + + if (data.size() >= maxSize) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it's exists, it becomes the + most recently used item. */ + // FIXME: use boost::optional? + Value * get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return 0; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return &i->second.second; + } + + size_t size() + { + return data.size(); + } +}; + +}