From 0e254ca66de4fa21ce5715df57d69c5e4a543dd7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2016 14:06:17 +0100 Subject: [PATCH] Refactor local binary cache code into a subclass --- src/hydra-queue-runner/Makefile.am | 3 +- src/hydra-queue-runner/binary-cache-store.cc | 45 +++++++------------ src/hydra-queue-runner/binary-cache-store.hh | 20 ++++++--- src/hydra-queue-runner/hydra-queue-runner.cc | 6 ++- .../local-binary-cache-store.cc | 43 ++++++++++++++++++ .../local-binary-cache-store.hh | 30 +++++++++++++ 6 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 src/hydra-queue-runner/local-binary-cache-store.cc create mode 100644 src/hydra-queue-runner/local-binary-cache-store.hh diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 9303d7d5..3b5b48a2 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -3,7 +3,8 @@ bin_PROGRAMS = hydra-queue-runner hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \ - binary-cache-store.hh binary-cache-store.cc + binary-cache-store.hh binary-cache-store.cc \ + local-binary-cache-store.hh local-binary-cache-store.cc hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx AM_CXXFLAGS = $(NIX_CFLAGS) -Wall diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc index 4fe11007..80720ba0 100644 --- a/src/hydra-queue-runner/binary-cache-store.cc +++ b/src/hydra-queue-runner/binary-cache-store.cc @@ -9,17 +9,10 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(ref localStore, const Path & binaryCacheDir, +BinaryCacheStore::BinaryCacheStore(ref localStore, const Path & secretKeyFile, const Path & publicKeyFile) : localStore(localStore) - , binaryCacheDir(binaryCacheDir) { - createDirs(binaryCacheDir + "/nar"); - - Path cacheInfoFile = binaryCacheDir + "/nix-cache-info"; - if (!pathExists(cacheInfoFile)) - writeFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n"); - if (secretKeyFile != "") secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); @@ -30,27 +23,24 @@ BinaryCacheStore::BinaryCacheStore(ref localStore, const Path & binaryCac } } +void BinaryCacheStore::init() +{ + std::string cacheInfoFile = "nix-cache-info"; + if (!fileExists(cacheInfoFile)) + upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n"); +} + Path BinaryCacheStore::narInfoFileFor(const Path & storePath) { assertStorePath(storePath); - return binaryCacheDir + "/" + storePathToHash(storePath) + ".narinfo"; -} - -void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); - del.cancel(); + return storePathToHash(storePath) + ".narinfo"; } void BinaryCacheStore::addToCache(const ValidPathInfo & info, const string & nar) { - Path narInfoFile = narInfoFileFor(info.path); - if (pathExists(narInfoFile)) return; + auto narInfoFile = narInfoFileFor(info.path); + if (fileExists(narInfoFile)) return; NarInfo narInfo(info); @@ -71,19 +61,18 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, /* Atomically write the NAR file. */ narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz"; - Path narFile = binaryCacheDir + "/" + narInfo.url; - if (!pathExists(narFile)) atomicWrite(narFile, narXz); + if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz); /* Atomically write the NAR info file.*/ if (secretKey) narInfo.sign(*secretKey); - atomicWrite(narInfoFile, narInfo.to_string()); + upsertFile(narInfoFile, narInfo.to_string()); } NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) { - Path narInfoFile = narInfoFileFor(storePath); - NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile); + auto narInfoFile = narInfoFileFor(storePath); + auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile); assert(narInfo.path == storePath); if (publicKeys) { @@ -96,7 +85,7 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) bool BinaryCacheStore::isValidPath(const Path & storePath) { - return pathExists(narInfoFileFor(storePath)); + return fileExists(narInfoFileFor(storePath)); } void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) @@ -105,7 +94,7 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink auto res = readNarInfo(storePath); - auto nar = readFile(binaryCacheDir + "/" + res.url); + auto nar = getFile(res.url); /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh index ac4f2fd7..0db7ddc4 100644 --- a/src/hydra-queue-runner/binary-cache-store.hh +++ b/src/hydra-queue-runner/binary-cache-store.hh @@ -7,23 +7,33 @@ namespace nix { struct NarInfo; -class BinaryCacheStore : public nix::Store +class BinaryCacheStore : public Store { private: + ref localStore; - Path binaryCacheDir; std::unique_ptr secretKey; std::unique_ptr publicKeys; +protected: + + BinaryCacheStore(ref localStore, + const Path & secretKeyFile, const Path & publicKeyFile); + + virtual bool fileExists(const std::string & path) = 0; + + virtual void upsertFile(const std::string & path, const std::string & data) = 0; + + virtual std::string getFile(const std::string & path) = 0; + public: - BinaryCacheStore(ref localStore, const Path & binaryCacheDir, - const Path & secretKeyFile, const Path & publicKeyFile); + virtual void init(); private: - Path narInfoFileFor(const Path & storePath); + std::string narInfoFileFor(const Path & storePath); void addToCache(const ValidPathInfo & info, const string & nar); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 9034850e..8f363ef3 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -7,7 +7,7 @@ #include "state.hh" #include "build-result.hh" -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "shared.hh" #include "globals.hh" @@ -33,10 +33,12 @@ ref State::getLocalStore() ref State::getDestStore() { - return make_ref(getLocalStore(), + auto store = make_ref(getLocalStore(), "/tmp/binary-cache", "/home/eelco/Misc/Keys/test.nixos.org/secret", "/home/eelco/Misc/Keys/test.nixos.org/public"); + store->init(); + return store; } diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc new file mode 100644 index 00000000..f0b2637f --- /dev/null +++ b/src/hydra-queue-runner/local-binary-cache-store.cc @@ -0,0 +1,43 @@ +#include "local-binary-cache-store.hh" + +namespace nix { + +LocalBinaryCacheStore::LocalBinaryCacheStore(ref localStore, + const Path & binaryCacheDir, const Path & secretKeyFile, const Path & publicKeyFile) + : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) + , binaryCacheDir(binaryCacheDir) +{ +} + +void LocalBinaryCacheStore::init() +{ + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); +} + +static void atomicWrite(const Path & path, const std::string & s) +{ + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string & path) +{ + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +{ + atomicWrite(binaryCacheDir + "/" + path, data); +} + +std::string LocalBinaryCacheStore::getFile(const std::string & path) +{ + return readFile(binaryCacheDir + "/" + path); +} + +} diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh new file mode 100644 index 00000000..91c56abc --- /dev/null +++ b/src/hydra-queue-runner/local-binary-cache-store.hh @@ -0,0 +1,30 @@ +#pragma once + +#include "binary-cache-store.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore(ref localStore, const Path & binaryCacheDir, + const Path & secretKeyFile, const Path & publicKeyFile); + + void init() override; + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + +}