From bd76f9120a1c600e65fcfcc9ba3d98c0e91cc80a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2016 16:19:40 +0100 Subject: [PATCH] Cache .narinfo lookups --- src/hydra-queue-runner/binary-cache-store.cc | 24 +++++- src/hydra-queue-runner/binary-cache-store.hh | 12 +++ src/hydra-queue-runner/hydra-queue-runner.cc | 2 + src/hydra-queue-runner/lru-cache.hh | 80 ++++++++++++++++++++ src/hydra-queue-runner/sync.hh | 2 +- 5 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 src/hydra-queue-runner/lru-cache.hh diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc index db25b7c6..226c8dda 100644 --- a/src/hydra-queue-runner/binary-cache-store.cc +++ b/src/hydra-queue-runner/binary-cache-store.cc @@ -1,4 +1,5 @@ #include "binary-cache-store.hh" +#include "sync.hh" #include "archive.hh" #include "compression.hh" @@ -93,18 +94,33 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) { + { + auto state_(state.lock()); + auto res = state_->narInfoCache.get(storePath); + if (res) { + stats.narInfoReadAverted++; + return **res; + } + } + stats.narInfoRead++; auto narInfoFile = narInfoFileFor(storePath); - auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile); - assert(narInfo.path == storePath); + auto narInfo = make_ref(getFile(narInfoFile), narInfoFile); + assert(narInfo->path == storePath); if (publicKeys) { - if (!narInfo.checkSignature(*publicKeys)) + if (!narInfo->checkSignature(*publicKeys)) throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile); } - return narInfo; + { + auto state_(state.lock()); + state_->narInfoCache.upsert(storePath, narInfo); + stats.narInfoCacheSize = state_->narInfoCache.size(); + } + + return *narInfo; } bool BinaryCacheStore::isValidPath(const Path & storePath) diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh index 019b2611..709a18cc 100644 --- a/src/hydra-queue-runner/binary-cache-store.hh +++ b/src/hydra-queue-runner/binary-cache-store.hh @@ -3,6 +3,9 @@ #include "crypto.hh" #include "store-api.hh" +#include "lru-cache.hh" +#include "sync.hh" + #include namespace nix { @@ -23,6 +26,13 @@ private: StoreFactory storeFactory; + struct State + { + LRUCache> narInfoCache{32 * 1024}; + }; + + Sync state; + protected: BinaryCacheStore(const StoreFactory & storeFactory, @@ -41,7 +51,9 @@ public: struct Stats { std::atomic narInfoRead{0}; + std::atomic narInfoReadAverted{0}; std::atomic narInfoWrite{0}; + std::atomic narInfoCacheSize{0}; std::atomic narRead{0}; std::atomic narReadBytes{0}; std::atomic narReadCompressedBytes{0}; diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 48c6865a..8080af7a 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -591,7 +591,9 @@ void State::dumpStatus(Connection & conn, bool log) auto & stats = store->getStats(); nested.attr("narInfoRead", stats.narInfoRead); + nested.attr("narInfoReadAverted", stats.narInfoReadAverted); nested.attr("narInfoWrite", stats.narInfoWrite); + nested.attr("narInfoCacheSize", stats.narInfoCacheSize); nested.attr("narRead", stats.narRead); nested.attr("narReadBytes", stats.narReadBytes); nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes); diff --git a/src/hydra-queue-runner/lru-cache.hh b/src/hydra-queue-runner/lru-cache.hh new file mode 100644 index 00000000..113411bd --- /dev/null +++ b/src/hydra-queue-runner/lru-cache.hh @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + +/* A simple least-recently used cache. Not thread-safe. */ +template +class LRUCache +{ +private: + + size_t maxSize; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map>; + using LRU = std::list; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t maxSize) : maxSize(maxSize) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + erase(key); + + if (data.size() >= maxSize) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it's exists, it becomes the + most recently used item. */ + // FIXME: use boost::optional? + Value * get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return 0; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return &i->second.second; + } + + size_t size() + { + return data.size(); + } +}; diff --git a/src/hydra-queue-runner/sync.hh b/src/hydra-queue-runner/sync.hh index 1573f091..4d53a6ee 100644 --- a/src/hydra-queue-runner/sync.hh +++ b/src/hydra-queue-runner/sync.hh @@ -20,7 +20,7 @@ scope. */ -template +template class Sync { private: