snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Refactor local binary cache code into a subclass

Browse source
This commit is contained in:
Eelco Dolstra 2016-02-18 14:06:17 +01:00
parent a992f688d1
commit 0e254ca66d
6 changed files with 111 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/hydra-queue-runner/Makefile.am
View file

@ -3,7 +3,8 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
binary-cache-store.hh binary-cache-store.cc
binary-cache-store.hh binary-cache-store.cc \
local-binary-cache-store.hh local-binary-cache-store.cc
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
AM_CXXFLAGS = $(NIX_CFLAGS) -Wall

45
src/hydra-queue-runner/binary-cache-store.cc
View file

@ -9,17 +9,10 @@
namespace nix {
BinaryCacheStore::BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
BinaryCacheStore::BinaryCacheStore(ref<Store> localStore,
const Path & secretKeyFile, const Path & publicKeyFile)
: localStore(localStore)
, binaryCacheDir(binaryCacheDir)
{
createDirs(binaryCacheDir + "/nar");
Path cacheInfoFile = binaryCacheDir + "/nix-cache-info";
if (!pathExists(cacheInfoFile))
writeFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@ -30,27 +23,24 @@ BinaryCacheStore::BinaryCacheStore(ref<Store> localStore, const Path & binaryCac
}
}
void BinaryCacheStore::init()
{
std::string cacheInfoFile = "nix-cache-info";
if (!fileExists(cacheInfoFile))
upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
}
Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
{
assertStorePath(storePath);
return binaryCacheDir + "/" + storePathToHash(storePath) + ".narinfo";
}
void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
del.cancel();
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::addToCache(const ValidPathInfo & info,
const string & nar)
{
Path narInfoFile = narInfoFileFor(info.path);
if (pathExists(narInfoFile)) return;
auto narInfoFile = narInfoFileFor(info.path);
if (fileExists(narInfoFile)) return;
NarInfo narInfo(info);
@ -71,19 +61,18 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
/* Atomically write the NAR file. */
narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
Path narFile = binaryCacheDir + "/" + narInfo.url;
if (!pathExists(narFile)) atomicWrite(narFile, narXz);
if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz);
/* Atomically write the NAR info file.*/
if (secretKey) narInfo.sign(*secretKey);
atomicWrite(narInfoFile, narInfo.to_string());
upsertFile(narInfoFile, narInfo.to_string());
}
NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
{
Path narInfoFile = narInfoFileFor(storePath);
NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile);
auto narInfoFile = narInfoFileFor(storePath);
auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile);
assert(narInfo.path == storePath);
if (publicKeys) {
@ -96,7 +85,7 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
bool BinaryCacheStore::isValidPath(const Path & storePath)
{
return pathExists(narInfoFileFor(storePath));
return fileExists(narInfoFileFor(storePath));
}
void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)
@ -105,7 +94,7 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
auto res = readNarInfo(storePath);
auto nar = readFile(binaryCacheDir + "/" + res.url);
auto nar = getFile(res.url);
/* Decompress the NAR. FIXME: would be nice to have the remote
side do this. */

20
src/hydra-queue-runner/binary-cache-store.hh
View file

@ -7,23 +7,33 @@ namespace nix {
struct NarInfo;
class BinaryCacheStore : public nix::Store
class BinaryCacheStore : public Store
{
private:
ref<Store> localStore;
Path binaryCacheDir;
std::unique_ptr<SecretKey> secretKey;
std::unique_ptr<PublicKeys> publicKeys;
protected:
BinaryCacheStore(ref<Store> localStore,
const Path & secretKeyFile, const Path & publicKeyFile);
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path, const std::string & data) = 0;
virtual std::string getFile(const std::string & path) = 0;
public:
BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
const Path & secretKeyFile, const Path & publicKeyFile);
virtual void init();
private:
Path narInfoFileFor(const Path & storePath);
std::string narInfoFileFor(const Path & storePath);
void addToCache(const ValidPathInfo & info, const string & nar);

6
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -7,7 +7,7 @@
#include "state.hh"
#include "build-result.hh"
#include "binary-cache-store.hh"
#include "local-binary-cache-store.hh"
#include "shared.hh"
#include "globals.hh"
@ -33,10 +33,12 @@ ref<Store> State::getLocalStore()
ref<Store> State::getDestStore()
{
return make_ref<BinaryCacheStore>(getLocalStore(),
auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
"/tmp/binary-cache",
"/home/eelco/Misc/Keys/test.nixos.org/secret",
"/home/eelco/Misc/Keys/test.nixos.org/public");
store->init();
return store;
}

43
src/hydra-queue-runner/local-binary-cache-store.cc Normal file
View file

@ -0,0 +1,43 @@
#include "local-binary-cache-store.hh"
namespace nix {
LocalBinaryCacheStore::LocalBinaryCacheStore(ref<Store> localStore,
const Path & binaryCacheDir, const Path & secretKeyFile, const Path & publicKeyFile)
: BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
, binaryCacheDir(binaryCacheDir)
{
}
void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
BinaryCacheStore::init();
}
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
}
void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
std::string LocalBinaryCacheStore::getFile(const std::string & path)
{
return readFile(binaryCacheDir + "/" + path);
}
}

30
src/hydra-queue-runner/local-binary-cache-store.hh Normal file
View file

@ -0,0 +1,30 @@
#pragma once
#include "binary-cache-store.hh"
namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore
{
private:
Path binaryCacheDir;
public:
LocalBinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
const Path & secretKeyFile, const Path & publicKeyFile);
void init() override;
protected:
bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path, const std::string & data) override;
std::string getFile(const std::string & path) override;
};
}