From 2d0dd7fb4963119b4ee26ec093d6910fd0d08d23 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 15 Feb 2016 21:10:29 +0100
Subject: [PATCH 01/40] hydra-queue-runner: Write directly to a binary cache

---
 src/hydra-queue-runner/Makefile.am           |   3 +-
 src/hydra-queue-runner/build-remote.cc       |  32 ++-
 src/hydra-queue-runner/build-result.cc       |   6 +
 src/hydra-queue-runner/builder.cc            |  10 +-
 src/hydra-queue-runner/hydra-queue-runner.cc |  16 +-
 src/hydra-queue-runner/local-binary-cache.cc | 267 +++++++++++++++++++
 src/hydra-queue-runner/local-binary-cache.hh | 124 +++++++++
 src/hydra-queue-runner/queue-monitor.cc      |  25 +-
 src/hydra-queue-runner/state.hh              |  14 +-
 9 files changed, 463 insertions(+), 34 deletions(-)
 create mode 100644 src/hydra-queue-runner/local-binary-cache.cc
 create mode 100644 src/hydra-queue-runner/local-binary-cache.hh

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 089187d1..21375f1c 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -2,7 +2,8 @@ bin_PROGRAMS = hydra-queue-runner
 
 hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
  builder.cc build-result.cc build-remote.cc  \
- build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh
+ build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
+ local-binary-cache.hh local-binary-cache.cc
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall
diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 39d041d9..11490432 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -73,14 +73,14 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
 }
 
 
-static void copyClosureTo(ref<Store> store,
+static void copyClosureTo(ref<Store> destStore,
     FdSource & from, FdSink & to, const PathSet & paths,
     counter & bytesSent,
     bool useSubstitutes = false)
 {
     PathSet closure;
     for (auto & path : paths)
-        store->computeFSClosure(path, closure);
+        destStore->computeFSClosure(path, closure);
 
     /* Send the "query valid paths" command with the "lock" option
        enabled. This prevents a race where the remote host
@@ -95,7 +95,7 @@ static void copyClosureTo(ref<Store> store,
 
     if (present.size() == closure.size()) return;
 
-    Paths sorted = store->topoSortPaths(closure);
+    Paths sorted = destStore->topoSortPaths(closure);
 
     Paths missing;
     for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
@@ -104,10 +104,10 @@ static void copyClosureTo(ref<Store> store,
     printMsg(lvlDebug, format("sending %1% missing paths") % missing.size());
 
     for (auto & p : missing)
-        bytesSent += store->queryPathInfo(p).narSize;
+        bytesSent += destStore->queryPathInfo(p).narSize;
 
     to << cmdImportPaths;
-    store->exportPaths(missing, false, to);
+    destStore->exportPaths(missing, false, to);
     to.flush();
 
     if (readInt(from) != 1)
@@ -115,19 +115,19 @@ static void copyClosureTo(ref<Store> store,
 }
 
 
-static void copyClosureFrom(ref<Store> store,
+static void copyClosureFrom(ref<Store> destStore,
     FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived)
 {
     to << cmdExportPaths << 0 << paths;
     to.flush();
-    store->importPaths(false, from);
+    destStore->importPaths(false, from);
 
     for (auto & p : paths)
-        bytesReceived += store->queryPathInfo(p).narSize;
+        bytesReceived += destStore->queryPathInfo(p).narSize;
 }
 
 
-void State::buildRemote(ref<Store> store,
+void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout,
     RemoteResult & result)
@@ -222,14 +222,20 @@ void State::buildRemote(ref<Store> store,
         }
     }
 
+    /* Ensure that the inputs exist in the destination store. This is
+       a no-op for regular stores, but for the binary cache store,
+       this will copy the inputs to the binary cache from the local
+       store. */
+    destStore->buildPaths(basicDrv.inputSrcs);
+
     /* Copy the input closure. */
-    if (machine->sshName != "localhost") {
+    if (/* machine->sshName != "localhost" */ true) {
         auto mc1 = std::make_shared<MaintainCount>(nrStepsWaiting);
         std::lock_guard<std::mutex> sendLock(machine->state->sendLock);
         mc1.reset();
         MaintainCount mc2(nrStepsCopyingTo);
         printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName);
-        copyClosureTo(store, from, to, inputs, bytesSent);
+        copyClosureTo(destStore, from, to, inputs, bytesSent);
     }
 
     autoDelete.cancel();
@@ -277,13 +283,13 @@ void State::buildRemote(ref<Store> store,
     }
 
     /* Copy the output paths. */
-    if (machine->sshName != "localhost") {
+    if (/* machine->sshName != "localhost" */ true) {
         printMsg(lvlDebug, format("copying outputs of ‘%1%’ from ‘%2%’") % step->drvPath % machine->sshName);
         PathSet outputs;
         for (auto & output : step->drv.outputs)
             outputs.insert(output.second.path);
         MaintainCount mc(nrStepsCopyingFrom);
-        copyClosureFrom(store, from, to, outputs, bytesReceived);
+        copyClosureFrom(destStore, from, to, outputs, bytesReceived);
     }
 
     /* Shut down the connection. */
diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index ebf07035..f09d13a1 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -41,6 +41,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
     /* Get build products. */
     bool explicitProducts = false;
 
+#if 0
     Regex regex(
         "(([a-zA-Z0-9_-]+)" // type (e.g. "doc")
         "[[:space:]]+"
@@ -97,6 +98,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
             res.products.push_back(product);
         }
     }
+#endif
 
     /* If no build products were explicitly declared, then add all
        outputs as a product of type "nix-build". */
@@ -108,14 +110,17 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
             product.subtype = output.first == "out" ? "" : output.first;
             product.name = storePathToName(product.path);
 
+#if 0
             struct stat st;
             if (stat(product.path.c_str(), &st))
                 throw SysError(format("getting status of ‘%1%’") % product.path);
             if (S_ISDIR(st.st_mode))
+#endif
                 res.products.push_back(product);
         }
     }
 
+#if 0
     /* Get the release name from $output/nix-support/hydra-release-name. */
     for (auto & output : outputs) {
         Path p = output + "/nix-support/hydra-release-name";
@@ -139,6 +144,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
             res.metrics[metric.name] = metric;
         }
     }
+#endif
 
     return res;
 }
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 9d1fd0dc..0c564fa2 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -15,8 +15,8 @@ void State::builder(MachineReservation::ptr reservation)
     auto step = reservation->step;
 
     try {
-        auto store = openStore(); // FIXME: pool
-        retry = doBuildStep(store, step, reservation->machine);
+        auto destStore = getDestStore();
+        retry = doBuildStep(destStore, step, reservation->machine);
     } catch (std::exception & e) {
         printMsg(lvlError, format("uncaught exception building ‘%1%’ on ‘%2%’: %3%")
             % step->drvPath % reservation->machine->sshName % e.what());
@@ -45,7 +45,7 @@ void State::builder(MachineReservation::ptr reservation)
 }
 
 
-bool State::doBuildStep(nix::ref<Store> store, Step::ptr step,
+bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
     Machine::ptr machine)
 {
     {
@@ -120,13 +120,13 @@ bool State::doBuildStep(nix::ref<Store> store, Step::ptr step,
         /* Do the build. */
         try {
             /* FIXME: referring builds may have conflicting timeouts. */
-            buildRemote(store, machine, step, build->maxSilentTime, build->buildTimeout, result);
+            buildRemote(destStore, machine, step, build->maxSilentTime, build->buildTimeout, result);
         } catch (Error & e) {
             result.status = BuildResult::MiscFailure;
             result.errorMsg = e.msg();
         }
 
-        if (result.success()) res = getBuildOutput(store, step->drv);
+        if (result.success()) res = getBuildOutput(destStore, step->drv);
     }
 
     time_t stepStopTime = time(0);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 4d1146f5..1e896ce9 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -7,6 +7,7 @@
 
 #include "state.hh"
 #include "build-result.hh"
+#include "local-binary-cache.hh"
 
 #include "shared.hh"
 #include "globals.hh"
@@ -24,6 +25,18 @@ State::State()
 }
 
 
+ref<Store> State::getLocalStore()
+{
+    return openStore(); // FIXME: pool
+}
+
+
+ref<Store> State::getDestStore()
+{
+    return make_ref<LocalBinaryCache>(getLocalStore(), "/tmp/binary-cache");
+}
+
+
 void State::parseMachines(const std::string & contents)
 {
     Machines newMachines, oldMachines;
@@ -94,7 +107,8 @@ void State::monitorMachinesFile()
         getEnv("NIX_REMOTE_SYSTEMS", pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":");
 
     if (machinesFiles.empty()) {
-        parseMachines("localhost " + settings.thisSystem
+        parseMachines("localhost " +
+            (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem)
             + " - " + std::to_string(settings.maxBuildJobs) + " 1");
         return;
     }
diff --git a/src/hydra-queue-runner/local-binary-cache.cc b/src/hydra-queue-runner/local-binary-cache.cc
new file mode 100644
index 00000000..997f5a51
--- /dev/null
+++ b/src/hydra-queue-runner/local-binary-cache.cc
@@ -0,0 +1,267 @@
+#include "local-binary-cache.hh"
+
+#include "archive.hh"
+#include "derivations.hh"
+#include "globals.hh"
+#include "worker-protocol.hh"
+
+namespace nix {
+
+LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir)
+    : localStore(localStore), binaryCacheDir(binaryCacheDir)
+{
+    createDirs(binaryCacheDir + "/nar");
+}
+
+Path LocalBinaryCache::narInfoFileFor(const Path & storePath)
+{
+    assertStorePath(storePath);
+    return binaryCacheDir + "/" + storePathToHash(storePath) + ".narinfo";
+}
+
+void atomicWrite(const Path & path, const std::string & s)
+{
+    Path tmp = path + ".tmp." + std::to_string(getpid());
+    AutoDelete del(tmp, false);
+    writeFile(tmp, s);
+    if (rename(tmp.c_str(), path.c_str()))
+        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
+    del.cancel();
+}
+
+void LocalBinaryCache::addToCache(const ValidPathInfo & info,
+    const string & nar)
+{
+    size_t narSize = nar.size();
+    Hash narHash = hashString(htSHA256, nar);
+
+    if (info.hash.type != htUnknown && info.hash != narHash)
+        throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
+
+    printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
+        % info.path % narSize);
+
+    /* Atomically write the NAR file. */
+    string narFileRel = "nar/" + printHash(narHash) + ".nar";
+    Path narFile = binaryCacheDir + "/" + narFileRel;
+    if (!pathExists(narFile)) atomicWrite(narFile, nar);
+
+    /* Atomically write the NAR info file.*/
+    Path narInfoFile = narInfoFileFor(info.path);
+
+    if (!pathExists(narInfoFile)) {
+
+        Strings refs;
+        for (auto & r : info.references)
+            refs.push_back(baseNameOf(r));
+
+        std::string narInfo;
+        narInfo += "StorePath: " + info.path + "\n";
+        narInfo += "URL: " + narFileRel + "\n";
+        narInfo += "Compression: none\n";
+        narInfo += "FileHash: sha256:" + printHash(narHash) + "\n";
+        narInfo += "FileSize: " + std::to_string(narSize) + "\n";
+        narInfo += "NarHash: sha256:" + printHash(narHash) + "\n";
+        narInfo += "NarSize: " + std::to_string(narSize) + "\n";
+        narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
+
+        // FIXME: add signature
+
+        atomicWrite(narInfoFile, narInfo);
+    }
+}
+
+LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
+{
+    NarInfo res;
+
+    Path narInfoFile = narInfoFileFor(storePath);
+    if (!pathExists(narInfoFile))
+        abort();
+    std::string narInfo = readFile(narInfoFile);
+
+    auto corrupt = [&]() {
+        throw Error(format("corrupt NAR info file ‘%1%’") % narInfoFile);
+    };
+
+    size_t pos = 0;
+    while (pos < narInfo.size()) {
+
+        size_t colon = narInfo.find(':', pos);
+        if (colon == std::string::npos) corrupt();
+
+        std::string name(narInfo, pos, colon - pos);
+
+        size_t eol = narInfo.find('\n', colon + 2);
+        if (eol == std::string::npos) corrupt();
+
+        std::string value(narInfo, colon + 2, eol - colon - 2);
+
+        if (name == "StorePath") {
+            res.info.path = value;
+            if (value != storePath) corrupt();
+            res.info.path = value;
+        }
+        else if (name == "References") {
+            auto refs = tokenizeString<Strings>(value, " ");
+            if (!res.info.references.empty()) corrupt();
+            for (auto & r : refs)
+                res.info.references.insert(settings.nixStore + "/" + r);
+        }
+        else if (name == "URL") {
+            res.narUrl = value;
+        }
+
+        pos = eol + 1;
+    }
+
+    if (res.info.path.empty() || res.narUrl.empty()) corrupt();
+
+    return res;
+}
+
+bool LocalBinaryCache::isValidPath(const Path & storePath)
+{
+    Path narInfoFile = narInfoFileFor(storePath);
+
+    printMsg(lvlDebug, format("checking %1% -> %2%") % storePath % narInfoFile);
+
+    return pathExists(narInfoFile);
+}
+
+void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink)
+{
+    assert(!sign);
+
+    auto res = readNarInfo(storePath);
+
+    auto nar = readFile(binaryCacheDir + "/" + res.narUrl);
+
+    printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
+
+    assert(nar.size() % 8 == 0);
+
+    sink((unsigned char *) nar.c_str(), nar.size());
+
+    // FIXME: check integrity of NAR.
+
+    sink << exportMagic << storePath << res.info.references << res.info.deriver << 0;
+}
+
+Paths LocalBinaryCache::importPaths(bool requireSignature, Source & source)
+{
+    assert(!requireSignature);
+    Paths res;
+    while (true) {
+        unsigned long long n = readLongLong(source);
+        if (n == 0) break;
+        if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
+        res.push_back(importPath(source));
+    }
+    return res;
+}
+
+struct TeeSource : Source
+{
+    Source & readSource;
+    std::string data;
+    TeeSource(Source & readSource) : readSource(readSource)
+    {
+    }
+    size_t read(unsigned char * data, size_t len)
+    {
+        size_t n = readSource.read(data, len);
+        this->data.append((char *) data, n);
+        return n;
+    }
+};
+
+struct NopSink : ParseSink
+{
+};
+
+Path LocalBinaryCache::importPath(Source & source)
+{
+    /* FIXME: some cut&paste of LocalStore::importPath(). */
+
+    /* Extract the NAR from the source. */
+    TeeSource tee(source);
+    NopSink sink;
+    parseDump(sink, tee);
+
+    uint32_t magic = readInt(source);
+    if (magic != exportMagic)
+        throw Error("Nix archive cannot be imported; wrong format");
+
+    ValidPathInfo info;
+    info.path = readStorePath(source);
+
+    info.references = readStorePaths<PathSet>(source);
+
+    readString(source); // deriver, don't care
+
+    bool haveSignature = readInt(source) == 1;
+    assert(!haveSignature);
+
+    addToCache(info, tee.data);
+
+    return info.path;
+}
+
+ValidPathInfo LocalBinaryCache::queryPathInfo(const Path & storePath)
+{
+    return readNarInfo(storePath).info;
+}
+
+void LocalBinaryCache::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
+{
+    PathSet left;
+
+    for (auto & storePath : paths) {
+        if (!localStore->isValidPath(storePath)) {
+            left.insert(storePath);
+            continue;
+        }
+        ValidPathInfo info = localStore->queryPathInfo(storePath);
+        SubstitutablePathInfo sub;
+        sub.references = info.references;
+        sub.downloadSize = 0;
+        sub.narSize = info.narSize;
+        infos.emplace(storePath, sub);
+    }
+
+    localStore->querySubstitutablePathInfos(left, infos);
+}
+
+void LocalBinaryCache::buildPaths(const PathSet & paths, BuildMode buildMode)
+{
+    for (auto & storePath : paths) {
+        assert(!isDerivation(storePath));
+
+        if (isValidPath(storePath)) continue;
+
+        localStore->addTempRoot(storePath);
+
+        if (!localStore->isValidPath(storePath))
+            localStore->ensurePath(storePath);
+
+        ValidPathInfo info = localStore->queryPathInfo(storePath);
+
+        for (auto & ref : info.references)
+            if (ref != storePath)
+                ensurePath(ref);
+
+        StringSink sink;
+        dumpPath(storePath, sink);
+
+        addToCache(info, sink.s);
+    }
+}
+
+void LocalBinaryCache::ensurePath(const Path & path)
+{
+    buildPaths({path});
+}
+
+}
diff --git a/src/hydra-queue-runner/local-binary-cache.hh b/src/hydra-queue-runner/local-binary-cache.hh
new file mode 100644
index 00000000..4ee61f42
--- /dev/null
+++ b/src/hydra-queue-runner/local-binary-cache.hh
@@ -0,0 +1,124 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+class LocalBinaryCache : public nix::Store
+{
+private:
+    ref<Store> localStore;
+    Path binaryCacheDir;
+
+public:
+
+    LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir);
+
+private:
+
+    Path narInfoFileFor(const Path & storePath);
+
+    void addToCache(const ValidPathInfo & info, const string & nar);
+
+    struct NarInfo
+    {
+        ValidPathInfo info;
+        std::string narUrl;
+    };
+
+    NarInfo readNarInfo(const Path & storePath);
+
+public:
+
+    bool isValidPath(const Path & path) override;
+
+    PathSet queryValidPaths(const PathSet & paths) override
+    { abort(); }
+
+    PathSet queryAllValidPaths() override
+    { abort(); }
+
+    ValidPathInfo queryPathInfo(const Path & path) override;
+
+    Hash queryPathHash(const Path & path) override
+    { abort(); }
+
+    void queryReferrers(const Path & path,
+        PathSet & referrers) override
+    { abort(); }
+
+    Path queryDeriver(const Path & path) override
+    { abort(); }
+
+    PathSet queryValidDerivers(const Path & path) override
+    { abort(); }
+
+    PathSet queryDerivationOutputs(const Path & path) override
+    { abort(); }
+
+    StringSet queryDerivationOutputNames(const Path & path) override
+    { abort(); }
+
+    Path queryPathFromHashPart(const string & hashPart) override
+    { abort(); }
+
+    PathSet querySubstitutablePaths(const PathSet & paths) override
+    { abort(); }
+
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos) override;
+
+    Path addToStore(const string & name, const Path & srcPath,
+        bool recursive = true, HashType hashAlgo = htSHA256,
+        PathFilter & filter = defaultPathFilter, bool repair = false) override
+    { abort(); }
+
+    Path addTextToStore(const string & name, const string & s,
+        const PathSet & references, bool repair = false) override
+    { abort(); }
+
+    void exportPath(const Path & path, bool sign,
+        Sink & sink) override;
+
+    Paths importPaths(bool requireSignature, Source & source) override;
+
+    Path importPath(Source & source);
+
+    void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
+
+    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode = bmNormal) override
+    { abort(); }
+
+    void ensurePath(const Path & path) override;
+
+    void addTempRoot(const Path & path) override
+    { abort(); }
+
+    void addIndirectRoot(const Path & path) override
+    { abort(); }
+
+    void syncWithGC() override
+    { }
+
+    Roots findRoots() override
+    { abort(); }
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override
+    { abort(); }
+
+    PathSet queryFailedPaths() override
+    { return PathSet(); }
+
+    void clearFailedPaths(const PathSet & paths) override
+    { }
+
+    void optimiseStore() override
+    { }
+
+    bool verifyStore(bool checkContents, bool repair) override
+    { return true; }
+
+};
+
+}
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index bf96f489..f7e40827 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -30,12 +30,13 @@ void State::queueMonitorLoop()
     receiver buildsBumped(*conn, "builds_bumped");
     receiver jobsetSharesChanged(*conn, "jobset_shares_changed");
 
-    auto store = openStore(); // FIXME: pool
+    auto localStore = getLocalStore();
+    auto destStore = getDestStore();
 
     unsigned int lastBuildId = 0;
 
     while (true) {
-        bool done = getQueuedBuilds(*conn, store, lastBuildId);
+        bool done = getQueuedBuilds(*conn, localStore, destStore, lastBuildId);
 
         /* Sleep until we get notification from the database about an
            event. */
@@ -63,7 +64,8 @@ void State::queueMonitorLoop()
 }
 
 
-bool State::getQueuedBuilds(Connection & conn, ref<Store> store, unsigned int & lastBuildId)
+bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
+    ref<Store> destStore, unsigned int & lastBuildId)
 {
     printMsg(lvlInfo, format("checking the queue for builds > %1%...") % lastBuildId);
 
@@ -118,7 +120,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> store, unsigned int &
         nrAdded++;
         newBuildsByID.erase(build->id);
 
-        if (!store->isValidPath(build->drvPath)) {
+        if (!localStore->isValidPath(build->drvPath)) {
             /* Derivation has been GC'ed prematurely. */
             printMsg(lvlError, format("aborting GC'ed build %1%") % build->id);
             if (!build->finishedInDB) {
@@ -138,7 +140,8 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> store, unsigned int &
 
         std::set<Step::ptr> newSteps;
         std::set<Path> finishedDrvs; // FIXME: re-use?
-        Step::ptr step = createStep(store, conn, build, build->drvPath, build, 0, finishedDrvs, newSteps, newRunnable);
+        Step::ptr step = createStep(destStore, conn, build, build->drvPath,
+            build, 0, finishedDrvs, newSteps, newRunnable);
 
         /* Some of the new steps may be the top level of builds that
            we haven't processed yet. So do them now. This ensures that
@@ -156,7 +159,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> store, unsigned int &
            all valid. So we mark this as a finished, cached build. */
         if (!step) {
             Derivation drv = readDerivation(build->drvPath);
-            BuildOutput res = getBuildOutput(store, drv);
+            BuildOutput res = getBuildOutput(destStore, drv);
 
             pqxx::work txn(conn);
             time_t now = time(0);
@@ -314,7 +317,7 @@ void State::processQueueChange(Connection & conn)
 }
 
 
-Step::ptr State::createStep(ref<Store> store,
+Step::ptr State::createStep(ref<Store> destStore,
     Connection & conn, Build::ptr build, const Path & drvPath,
     Build::ptr referringBuild, Step::ptr referringStep, std::set<Path> & finishedDrvs,
     std::set<Step::ptr> & newSteps, std::set<Step::ptr> & newRunnable)
@@ -394,7 +397,7 @@ Step::ptr State::createStep(ref<Store> store,
     DerivationOutputs missing;
     PathSet missingPaths;
     for (auto & i : step->drv.outputs)
-        if (!store->isValidPath(i.second.path)) {
+        if (!destStore->isValidPath(i.second.path)) {
             valid = false;
             missing[i.first] = i.second;
             missingPaths.insert(i.second.path);
@@ -406,7 +409,7 @@ Step::ptr State::createStep(ref<Store> store,
     assert(missing.size() == missingPaths.size());
     if (!missing.empty() && settings.useSubstitutes) {
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos(missingPaths, infos);
+        destStore->querySubstitutablePathInfos(missingPaths, infos); // FIXME
         if (infos.size() == missingPaths.size()) {
             valid = true;
             for (auto & i : missing) {
@@ -414,7 +417,7 @@ Step::ptr State::createStep(ref<Store> store,
                     printMsg(lvlInfo, format("substituting output ‘%1%’ of ‘%2%’") % i.second.path % drvPath);
 
                     time_t startTime = time(0);
-                    store->ensurePath(i.second.path);
+                    destStore->ensurePath(i.second.path);
                     time_t stopTime = time(0);
 
                     {
@@ -443,7 +446,7 @@ Step::ptr State::createStep(ref<Store> store,
 
     /* Create steps for the dependencies. */
     for (auto & i : step->drv.inputDrvs) {
-        auto dep = createStep(store, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
+        auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable);
         if (dep) {
             auto step_(step->state.lock());
             step_->deps.insert(dep);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index a296ddc7..1b33edaa 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -350,6 +350,13 @@ public:
 
 private:
 
+    /* Return a store object that can access derivations produced by
+       hydra-evaluator. */
+    nix::ref<nix::Store> getLocalStore();
+
+    /* Return a store object to store build results. */
+    nix::ref<nix::Store> getDestStore();
+
     void clearBusy(Connection & conn, time_t stopTime);
 
     void parseMachines(const std::string & contents);
@@ -377,7 +384,8 @@ private:
     void queueMonitorLoop();
 
     /* Check the queue for new builds. */
-    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> store, unsigned int & lastBuildId);
+    bool getQueuedBuilds(Connection & conn, nix::ref<nix::Store> localStore,
+        nix::ref<nix::Store> destStore, unsigned int & lastBuildId);
 
     /* Handle cancellation, deletion and priority bumps. */
     void processQueueChange(Connection & conn);
@@ -405,10 +413,10 @@ private:
 
     /* Perform the given build step. Return true if the step is to be
        retried. */
-    bool doBuildStep(nix::ref<nix::Store> store, Step::ptr step,
+    bool doBuildStep(nix::ref<nix::Store> destStore, Step::ptr step,
         Machine::ptr machine);
 
-    void buildRemote(nix::ref<nix::Store> store,
+    void buildRemote(nix::ref<nix::Store> destStore,
         Machine::ptr machine, Step::ptr step,
         unsigned int maxSilentTime, unsigned int buildTimeout,
         RemoteResult & result);

From 744cee134ec3f3c34ce4f9f019fc677f4ecf3b19 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 15 Feb 2016 21:56:53 +0100
Subject: [PATCH 02/40] hydra-queue-runner: Compress binary cache NARs using xz

---
 src/hydra-queue-runner/local-binary-cache.cc | 57 ++++++++++++--------
 src/hydra-queue-runner/local-binary-cache.hh |  1 +
 2 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/src/hydra-queue-runner/local-binary-cache.cc b/src/hydra-queue-runner/local-binary-cache.cc
index 997f5a51..e12143a6 100644
--- a/src/hydra-queue-runner/local-binary-cache.cc
+++ b/src/hydra-queue-runner/local-binary-cache.cc
@@ -1,6 +1,7 @@
 #include "local-binary-cache.hh"
 
 #include "archive.hh"
+#include "compression.hh"
 #include "derivations.hh"
 #include "globals.hh"
 #include "worker-protocol.hh"
@@ -32,6 +33,9 @@ void atomicWrite(const Path & path, const std::string & s)
 void LocalBinaryCache::addToCache(const ValidPathInfo & info,
     const string & nar)
 {
+    Path narInfoFile = narInfoFileFor(info.path);
+    if (pathExists(narInfoFile)) return;
+
     size_t narSize = nar.size();
     Hash narHash = hashString(htSHA256, nar);
 
@@ -41,34 +45,33 @@ void LocalBinaryCache::addToCache(const ValidPathInfo & info,
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
         % info.path % narSize);
 
+    /* Compress the NAR. */
+    string narXz = compressXZ(nar);
+    Hash narXzHash = hashString(htSHA256, narXz);
+
     /* Atomically write the NAR file. */
-    string narFileRel = "nar/" + printHash(narHash) + ".nar";
+    string narFileRel = "nar/" + printHash32(narXzHash) + ".nar.xz";
     Path narFile = binaryCacheDir + "/" + narFileRel;
-    if (!pathExists(narFile)) atomicWrite(narFile, nar);
+    if (!pathExists(narFile)) atomicWrite(narFile, narXz);
 
     /* Atomically write the NAR info file.*/
-    Path narInfoFile = narInfoFileFor(info.path);
+    Strings refs;
+    for (auto & r : info.references)
+        refs.push_back(baseNameOf(r));
 
-    if (!pathExists(narInfoFile)) {
+    std::string narInfo;
+    narInfo += "StorePath: " + info.path + "\n";
+    narInfo += "URL: " + narFileRel + "\n";
+    narInfo += "Compression: xz\n";
+    narInfo += "FileHash: sha256:" + printHash32(narXzHash) + "\n";
+    narInfo += "FileSize: " + std::to_string(narXz.size()) + "\n";
+    narInfo += "NarHash: sha256:" + printHash32(narHash) + "\n";
+    narInfo += "NarSize: " + std::to_string(narSize) + "\n";
+    narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
 
-        Strings refs;
-        for (auto & r : info.references)
-            refs.push_back(baseNameOf(r));
+    // FIXME: add signature
 
-        std::string narInfo;
-        narInfo += "StorePath: " + info.path + "\n";
-        narInfo += "URL: " + narFileRel + "\n";
-        narInfo += "Compression: none\n";
-        narInfo += "FileHash: sha256:" + printHash(narHash) + "\n";
-        narInfo += "FileSize: " + std::to_string(narSize) + "\n";
-        narInfo += "NarHash: sha256:" + printHash(narHash) + "\n";
-        narInfo += "NarSize: " + std::to_string(narSize) + "\n";
-        narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
-
-        // FIXME: add signature
-
-        atomicWrite(narInfoFile, narInfo);
-    }
+    atomicWrite(narInfoFile, narInfo);
 }
 
 LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
@@ -111,6 +114,9 @@ LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
         else if (name == "URL") {
             res.narUrl = value;
         }
+        else if (name == "Compression") {
+            res.compression = value;
+        }
 
         pos = eol + 1;
     }
@@ -137,6 +143,15 @@ void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink
 
     auto nar = readFile(binaryCacheDir + "/" + res.narUrl);
 
+    /* Decompress the NAR. FIXME: would be nice to have the remote
+       side do this. */
+    if (res.compression == "none")
+        ;
+    else if (res.compression == "xz")
+        nar = decompressXZ(nar);
+    else
+        throw Error(format("unknown NAR compression type ‘%1%’") % nar);
+
     printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
 
     assert(nar.size() % 8 == 0);
diff --git a/src/hydra-queue-runner/local-binary-cache.hh b/src/hydra-queue-runner/local-binary-cache.hh
index 4ee61f42..95c17e05 100644
--- a/src/hydra-queue-runner/local-binary-cache.hh
+++ b/src/hydra-queue-runner/local-binary-cache.hh
@@ -24,6 +24,7 @@ private:
     {
         ValidPathInfo info;
         std::string narUrl;
+        std::string compression = "none";
     };
 
     NarInfo readNarInfo(const Path & storePath);

From 25022bf5fdc1e0ed2eb2079105a21354758c05cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 16 Feb 2016 16:41:42 +0100
Subject: [PATCH 03/40] hydra-queue-runner: Support generating a signed binary
 cache

---
 src/hydra-queue-runner/hydra-queue-runner.cc |   5 +-
 src/hydra-queue-runner/local-binary-cache.cc | 120 +++++++------------
 src/hydra-queue-runner/local-binary-cache.hh |  16 +--
 3 files changed, 54 insertions(+), 87 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 1e896ce9..e594ec3b 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -33,7 +33,10 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
-    return make_ref<LocalBinaryCache>(getLocalStore(), "/tmp/binary-cache");
+    return make_ref<LocalBinaryCache>(getLocalStore(),
+        "/tmp/binary-cache",
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public");
 }
 
 
diff --git a/src/hydra-queue-runner/local-binary-cache.cc b/src/hydra-queue-runner/local-binary-cache.cc
index e12143a6..8b1d2431 100644
--- a/src/hydra-queue-runner/local-binary-cache.cc
+++ b/src/hydra-queue-runner/local-binary-cache.cc
@@ -4,14 +4,30 @@
 #include "compression.hh"
 #include "derivations.hh"
 #include "globals.hh"
+#include "nar-info.hh"
 #include "worker-protocol.hh"
 
 namespace nix {
 
-LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir)
-    : localStore(localStore), binaryCacheDir(binaryCacheDir)
+LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,
+    const Path & secretKeyFile, const Path & publicKeyFile)
+    : localStore(localStore)
+    , binaryCacheDir(binaryCacheDir)
 {
     createDirs(binaryCacheDir + "/nar");
+
+    Path cacheInfoFile = binaryCacheDir + "/nix-cache-info";
+    if (!pathExists(cacheInfoFile))
+        writeFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
+
+    if (secretKeyFile != "")
+        secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
+
+    if (publicKeyFile != "") {
+        publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);
+        auto key = PublicKey(readFile(publicKeyFile));
+        publicKeys->emplace(key.name, key);
+    }
 }
 
 Path LocalBinaryCache::narInfoFileFor(const Path & storePath)
@@ -36,103 +52,51 @@ void LocalBinaryCache::addToCache(const ValidPathInfo & info,
     Path narInfoFile = narInfoFileFor(info.path);
     if (pathExists(narInfoFile)) return;
 
-    size_t narSize = nar.size();
-    Hash narHash = hashString(htSHA256, nar);
+    NarInfo narInfo(info);
 
-    if (info.hash.type != htUnknown && info.hash != narHash)
+    narInfo.narSize = nar.size();
+    narInfo.narHash = hashString(htSHA256, nar);
+
+    if (info.narHash.type != htUnknown && info.narHash != narInfo.narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
-        % info.path % narSize);
+        % info.path % info.narSize);
 
     /* Compress the NAR. */
+    narInfo.compression = "xz";
     string narXz = compressXZ(nar);
-    Hash narXzHash = hashString(htSHA256, narXz);
+    narInfo.fileHash = hashString(htSHA256, narXz);
+    narInfo.fileSize = narXz.size();
 
     /* Atomically write the NAR file. */
-    string narFileRel = "nar/" + printHash32(narXzHash) + ".nar.xz";
-    Path narFile = binaryCacheDir + "/" + narFileRel;
+    narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
+    Path narFile = binaryCacheDir + "/" + narInfo.url;
     if (!pathExists(narFile)) atomicWrite(narFile, narXz);
 
     /* Atomically write the NAR info file.*/
-    Strings refs;
-    for (auto & r : info.references)
-        refs.push_back(baseNameOf(r));
+    if (secretKey) narInfo.sign(*secretKey);
 
-    std::string narInfo;
-    narInfo += "StorePath: " + info.path + "\n";
-    narInfo += "URL: " + narFileRel + "\n";
-    narInfo += "Compression: xz\n";
-    narInfo += "FileHash: sha256:" + printHash32(narXzHash) + "\n";
-    narInfo += "FileSize: " + std::to_string(narXz.size()) + "\n";
-    narInfo += "NarHash: sha256:" + printHash32(narHash) + "\n";
-    narInfo += "NarSize: " + std::to_string(narSize) + "\n";
-    narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
-
-    // FIXME: add signature
-
-    atomicWrite(narInfoFile, narInfo);
+    atomicWrite(narInfoFile, narInfo.to_string());
 }
 
-LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
+NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
 {
-    NarInfo res;
-
     Path narInfoFile = narInfoFileFor(storePath);
-    if (!pathExists(narInfoFile))
-        abort();
-    std::string narInfo = readFile(narInfoFile);
+    NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile);
+    assert(narInfo.path == storePath);
 
-    auto corrupt = [&]() {
-        throw Error(format("corrupt NAR info file ‘%1%’") % narInfoFile);
-    };
-
-    size_t pos = 0;
-    while (pos < narInfo.size()) {
-
-        size_t colon = narInfo.find(':', pos);
-        if (colon == std::string::npos) corrupt();
-
-        std::string name(narInfo, pos, colon - pos);
-
-        size_t eol = narInfo.find('\n', colon + 2);
-        if (eol == std::string::npos) corrupt();
-
-        std::string value(narInfo, colon + 2, eol - colon - 2);
-
-        if (name == "StorePath") {
-            res.info.path = value;
-            if (value != storePath) corrupt();
-            res.info.path = value;
-        }
-        else if (name == "References") {
-            auto refs = tokenizeString<Strings>(value, " ");
-            if (!res.info.references.empty()) corrupt();
-            for (auto & r : refs)
-                res.info.references.insert(settings.nixStore + "/" + r);
-        }
-        else if (name == "URL") {
-            res.narUrl = value;
-        }
-        else if (name == "Compression") {
-            res.compression = value;
-        }
-
-        pos = eol + 1;
+    if (publicKeys) {
+        if (!narInfo.checkSignature(*publicKeys))
+            throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
     }
 
-    if (res.info.path.empty() || res.narUrl.empty()) corrupt();
-
-    return res;
+    return narInfo;
 }
 
 bool LocalBinaryCache::isValidPath(const Path & storePath)
 {
-    Path narInfoFile = narInfoFileFor(storePath);
-
-    printMsg(lvlDebug, format("checking %1% -> %2%") % storePath % narInfoFile);
-
-    return pathExists(narInfoFile);
+    return pathExists(narInfoFileFor(storePath));
 }
 
 void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink)
@@ -141,7 +105,7 @@ void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink
 
     auto res = readNarInfo(storePath);
 
-    auto nar = readFile(binaryCacheDir + "/" + res.narUrl);
+    auto nar = readFile(binaryCacheDir + "/" + res.url);
 
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
@@ -160,7 +124,7 @@ void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink
 
     // FIXME: check integrity of NAR.
 
-    sink << exportMagic << storePath << res.info.references << res.info.deriver << 0;
+    sink << exportMagic << storePath << res.references << res.deriver << 0;
 }
 
 Paths LocalBinaryCache::importPaths(bool requireSignature, Source & source)
@@ -225,7 +189,7 @@ Path LocalBinaryCache::importPath(Source & source)
 
 ValidPathInfo LocalBinaryCache::queryPathInfo(const Path & storePath)
 {
-    return readNarInfo(storePath).info;
+    return ValidPathInfo(readNarInfo(storePath));
 }
 
 void LocalBinaryCache::querySubstitutablePathInfos(const PathSet & paths,
diff --git a/src/hydra-queue-runner/local-binary-cache.hh b/src/hydra-queue-runner/local-binary-cache.hh
index 95c17e05..0436974e 100644
--- a/src/hydra-queue-runner/local-binary-cache.hh
+++ b/src/hydra-queue-runner/local-binary-cache.hh
@@ -1,18 +1,25 @@
 #pragma once
 
+#include "crypto.hh"
 #include "store-api.hh"
 
 namespace nix {
 
+struct NarInfo;
+
 class LocalBinaryCache : public nix::Store
 {
 private:
     ref<Store> localStore;
     Path binaryCacheDir;
 
+    std::unique_ptr<SecretKey> secretKey;
+    std::unique_ptr<PublicKeys> publicKeys;
+
 public:
 
-    LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir);
+    LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,
+        const Path & secretKeyFile, const Path & publicKeyFile);
 
 private:
 
@@ -20,13 +27,6 @@ private:
 
     void addToCache(const ValidPathInfo & info, const string & nar);
 
-    struct NarInfo
-    {
-        ValidPathInfo info;
-        std::string narUrl;
-        std::string compression = "none";
-    };
-
     NarInfo readNarInfo(const Path & storePath);
 
 public:

From de77cc2910c56c6a39afa994b1ab0f7d43a995d8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 12:49:51 +0100
Subject: [PATCH 04/40] Rename file

---
 src/hydra-queue-runner/Makefile.am                              | 2 +-
 .../{local-binary-cache.cc => binary-cache-store.cc}            | 2 +-
 .../{local-binary-cache.hh => binary-cache-store.hh}            | 0
 src/hydra-queue-runner/hydra-queue-runner.cc                    | 2 +-
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename src/hydra-queue-runner/{local-binary-cache.cc => binary-cache-store.cc} (99%)
 rename src/hydra-queue-runner/{local-binary-cache.hh => binary-cache-store.hh} (100%)

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 21375f1c..9303d7d5 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -3,7 +3,7 @@ bin_PROGRAMS = hydra-queue-runner
 hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
  builder.cc build-result.cc build-remote.cc  \
  build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
- local-binary-cache.hh local-binary-cache.cc
+ binary-cache-store.hh binary-cache-store.cc
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall
diff --git a/src/hydra-queue-runner/local-binary-cache.cc b/src/hydra-queue-runner/binary-cache-store.cc
similarity index 99%
rename from src/hydra-queue-runner/local-binary-cache.cc
rename to src/hydra-queue-runner/binary-cache-store.cc
index 8b1d2431..e73cd00d 100644
--- a/src/hydra-queue-runner/local-binary-cache.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -1,4 +1,4 @@
-#include "local-binary-cache.hh"
+#include "binary-cache-store.hh"
 
 #include "archive.hh"
 #include "compression.hh"
diff --git a/src/hydra-queue-runner/local-binary-cache.hh b/src/hydra-queue-runner/binary-cache-store.hh
similarity index 100%
rename from src/hydra-queue-runner/local-binary-cache.hh
rename to src/hydra-queue-runner/binary-cache-store.hh
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index f3c7797b..ef68e484 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -7,7 +7,7 @@
 
 #include "state.hh"
 #include "build-result.hh"
-#include "local-binary-cache.hh"
+#include "binary-cache-store.hh"
 
 #include "shared.hh"
 #include "globals.hh"

From a992f688d1dac8a12e1cc83babe4c511dafa792f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 12:51:10 +0100
Subject: [PATCH 05/40] Rename class

---
 src/hydra-queue-runner/binary-cache-store.cc | 24 ++++++++++----------
 src/hydra-queue-runner/binary-cache-store.hh |  4 ++--
 src/hydra-queue-runner/hydra-queue-runner.cc |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index e73cd00d..4fe11007 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -9,7 +9,7 @@
 
 namespace nix {
 
-LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,
+BinaryCacheStore::BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
     const Path & secretKeyFile, const Path & publicKeyFile)
     : localStore(localStore)
     , binaryCacheDir(binaryCacheDir)
@@ -30,7 +30,7 @@ LocalBinaryCache::LocalBinaryCache(ref<Store> localStore, const Path & binaryCac
     }
 }
 
-Path LocalBinaryCache::narInfoFileFor(const Path & storePath)
+Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
 {
     assertStorePath(storePath);
     return binaryCacheDir + "/" + storePathToHash(storePath) + ".narinfo";
@@ -46,7 +46,7 @@ void atomicWrite(const Path & path, const std::string & s)
     del.cancel();
 }
 
-void LocalBinaryCache::addToCache(const ValidPathInfo & info,
+void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     const string & nar)
 {
     Path narInfoFile = narInfoFileFor(info.path);
@@ -80,7 +80,7 @@ void LocalBinaryCache::addToCache(const ValidPathInfo & info,
     atomicWrite(narInfoFile, narInfo.to_string());
 }
 
-NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
+NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 {
     Path narInfoFile = narInfoFileFor(storePath);
     NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile);
@@ -94,12 +94,12 @@ NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
     return narInfo;
 }
 
-bool LocalBinaryCache::isValidPath(const Path & storePath)
+bool BinaryCacheStore::isValidPath(const Path & storePath)
 {
     return pathExists(narInfoFileFor(storePath));
 }
 
-void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink)
+void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)
 {
     assert(!sign);
 
@@ -127,7 +127,7 @@ void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink
     sink << exportMagic << storePath << res.references << res.deriver << 0;
 }
 
-Paths LocalBinaryCache::importPaths(bool requireSignature, Source & source)
+Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
 {
     assert(!requireSignature);
     Paths res;
@@ -159,7 +159,7 @@ struct NopSink : ParseSink
 {
 };
 
-Path LocalBinaryCache::importPath(Source & source)
+Path BinaryCacheStore::importPath(Source & source)
 {
     /* FIXME: some cut&paste of LocalStore::importPath(). */
 
@@ -187,12 +187,12 @@ Path LocalBinaryCache::importPath(Source & source)
     return info.path;
 }
 
-ValidPathInfo LocalBinaryCache::queryPathInfo(const Path & storePath)
+ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
 {
     return ValidPathInfo(readNarInfo(storePath));
 }
 
-void LocalBinaryCache::querySubstitutablePathInfos(const PathSet & paths,
+void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
     SubstitutablePathInfos & infos)
 {
     PathSet left;
@@ -213,7 +213,7 @@ void LocalBinaryCache::querySubstitutablePathInfos(const PathSet & paths,
     localStore->querySubstitutablePathInfos(left, infos);
 }
 
-void LocalBinaryCache::buildPaths(const PathSet & paths, BuildMode buildMode)
+void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
     for (auto & storePath : paths) {
         assert(!isDerivation(storePath));
@@ -238,7 +238,7 @@ void LocalBinaryCache::buildPaths(const PathSet & paths, BuildMode buildMode)
     }
 }
 
-void LocalBinaryCache::ensurePath(const Path & path)
+void BinaryCacheStore::ensurePath(const Path & path)
 {
     buildPaths({path});
 }
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 0436974e..ac4f2fd7 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -7,7 +7,7 @@ namespace nix {
 
 struct NarInfo;
 
-class LocalBinaryCache : public nix::Store
+class BinaryCacheStore : public nix::Store
 {
 private:
     ref<Store> localStore;
@@ -18,7 +18,7 @@ private:
 
 public:
 
-    LocalBinaryCache(ref<Store> localStore, const Path & binaryCacheDir,
+    BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
         const Path & secretKeyFile, const Path & publicKeyFile);
 
 private:
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index ef68e484..9034850e 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -33,7 +33,7 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
-    return make_ref<LocalBinaryCache>(getLocalStore(),
+    return make_ref<BinaryCacheStore>(getLocalStore(),
         "/tmp/binary-cache",
         "/home/eelco/Misc/Keys/test.nixos.org/secret",
         "/home/eelco/Misc/Keys/test.nixos.org/public");

From 0e254ca66de4fa21ce5715df57d69c5e4a543dd7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 14:06:17 +0100
Subject: [PATCH 06/40] Refactor local binary cache code into a subclass

---
 src/hydra-queue-runner/Makefile.am            |  3 +-
 src/hydra-queue-runner/binary-cache-store.cc  | 45 +++++++------------
 src/hydra-queue-runner/binary-cache-store.hh  | 20 ++++++---
 src/hydra-queue-runner/hydra-queue-runner.cc  |  6 ++-
 .../local-binary-cache-store.cc               | 43 ++++++++++++++++++
 .../local-binary-cache-store.hh               | 30 +++++++++++++
 6 files changed, 111 insertions(+), 36 deletions(-)
 create mode 100644 src/hydra-queue-runner/local-binary-cache-store.cc
 create mode 100644 src/hydra-queue-runner/local-binary-cache-store.hh

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 9303d7d5..3b5b48a2 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -3,7 +3,8 @@ bin_PROGRAMS = hydra-queue-runner
 hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
  builder.cc build-result.cc build-remote.cc  \
  build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
- binary-cache-store.hh binary-cache-store.cc
+ binary-cache-store.hh binary-cache-store.cc \
+ local-binary-cache-store.hh local-binary-cache-store.cc
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall
diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index 4fe11007..80720ba0 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -9,17 +9,10 @@
 
 namespace nix {
 
-BinaryCacheStore::BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
+BinaryCacheStore::BinaryCacheStore(ref<Store> localStore,
     const Path & secretKeyFile, const Path & publicKeyFile)
     : localStore(localStore)
-    , binaryCacheDir(binaryCacheDir)
 {
-    createDirs(binaryCacheDir + "/nar");
-
-    Path cacheInfoFile = binaryCacheDir + "/nix-cache-info";
-    if (!pathExists(cacheInfoFile))
-        writeFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
-
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
 
@@ -30,27 +23,24 @@ BinaryCacheStore::BinaryCacheStore(ref<Store> localStore, const Path & binaryCac
     }
 }
 
+void BinaryCacheStore::init()
+{
+    std::string cacheInfoFile = "nix-cache-info";
+    if (!fileExists(cacheInfoFile))
+        upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
+}
+
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
 {
     assertStorePath(storePath);
-    return binaryCacheDir + "/" + storePathToHash(storePath) + ".narinfo";
-}
-
-void atomicWrite(const Path & path, const std::string & s)
-{
-    Path tmp = path + ".tmp." + std::to_string(getpid());
-    AutoDelete del(tmp, false);
-    writeFile(tmp, s);
-    if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
-    del.cancel();
+    return storePathToHash(storePath) + ".narinfo";
 }
 
 void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     const string & nar)
 {
-    Path narInfoFile = narInfoFileFor(info.path);
-    if (pathExists(narInfoFile)) return;
+    auto narInfoFile = narInfoFileFor(info.path);
+    if (fileExists(narInfoFile)) return;
 
     NarInfo narInfo(info);
 
@@ -71,19 +61,18 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
 
     /* Atomically write the NAR file. */
     narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
-    Path narFile = binaryCacheDir + "/" + narInfo.url;
-    if (!pathExists(narFile)) atomicWrite(narFile, narXz);
+    if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz);
 
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo.sign(*secretKey);
 
-    atomicWrite(narInfoFile, narInfo.to_string());
+    upsertFile(narInfoFile, narInfo.to_string());
 }
 
 NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 {
-    Path narInfoFile = narInfoFileFor(storePath);
-    NarInfo narInfo = NarInfo(readFile(narInfoFile), narInfoFile);
+    auto narInfoFile = narInfoFileFor(storePath);
+    auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile);
     assert(narInfo.path == storePath);
 
     if (publicKeys) {
@@ -96,7 +85,7 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 
 bool BinaryCacheStore::isValidPath(const Path & storePath)
 {
-    return pathExists(narInfoFileFor(storePath));
+    return fileExists(narInfoFileFor(storePath));
 }
 
 void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)
@@ -105,7 +94,7 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
 
     auto res = readNarInfo(storePath);
 
-    auto nar = readFile(binaryCacheDir + "/" + res.url);
+    auto nar = getFile(res.url);
 
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index ac4f2fd7..0db7ddc4 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -7,23 +7,33 @@ namespace nix {
 
 struct NarInfo;
 
-class BinaryCacheStore : public nix::Store
+class BinaryCacheStore : public Store
 {
 private:
+
     ref<Store> localStore;
-    Path binaryCacheDir;
 
     std::unique_ptr<SecretKey> secretKey;
     std::unique_ptr<PublicKeys> publicKeys;
 
+protected:
+
+    BinaryCacheStore(ref<Store> localStore,
+        const Path & secretKeyFile, const Path & publicKeyFile);
+
+    virtual bool fileExists(const std::string & path) = 0;
+
+    virtual void upsertFile(const std::string & path, const std::string & data) = 0;
+
+    virtual std::string getFile(const std::string & path) = 0;
+
 public:
 
-    BinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
-        const Path & secretKeyFile, const Path & publicKeyFile);
+    virtual void init();
 
 private:
 
-    Path narInfoFileFor(const Path & storePath);
+    std::string narInfoFileFor(const Path & storePath);
 
     void addToCache(const ValidPathInfo & info, const string & nar);
 
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 9034850e..8f363ef3 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -7,7 +7,7 @@
 
 #include "state.hh"
 #include "build-result.hh"
-#include "binary-cache-store.hh"
+#include "local-binary-cache-store.hh"
 
 #include "shared.hh"
 #include "globals.hh"
@@ -33,10 +33,12 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
-    return make_ref<BinaryCacheStore>(getLocalStore(),
+    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
         "/tmp/binary-cache",
         "/home/eelco/Misc/Keys/test.nixos.org/secret",
         "/home/eelco/Misc/Keys/test.nixos.org/public");
+    store->init();
+    return store;
 }
 
 
diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc
new file mode 100644
index 00000000..f0b2637f
--- /dev/null
+++ b/src/hydra-queue-runner/local-binary-cache-store.cc
@@ -0,0 +1,43 @@
+#include "local-binary-cache-store.hh"
+
+namespace nix {
+
+LocalBinaryCacheStore::LocalBinaryCacheStore(ref<Store> localStore,
+    const Path & binaryCacheDir,  const Path & secretKeyFile, const Path & publicKeyFile)
+    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    , binaryCacheDir(binaryCacheDir)
+{
+}
+
+void LocalBinaryCacheStore::init()
+{
+    createDirs(binaryCacheDir + "/nar");
+    BinaryCacheStore::init();
+}
+
+static void atomicWrite(const Path & path, const std::string & s)
+{
+    Path tmp = path + ".tmp." + std::to_string(getpid());
+    AutoDelete del(tmp, false);
+    writeFile(tmp, s);
+    if (rename(tmp.c_str(), path.c_str()))
+        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
+    del.cancel();
+}
+
+bool LocalBinaryCacheStore::fileExists(const std::string & path)
+{
+    return pathExists(binaryCacheDir + "/" + path);
+}
+
+void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
+{
+    atomicWrite(binaryCacheDir + "/" + path, data);
+}
+
+std::string LocalBinaryCacheStore::getFile(const std::string & path)
+{
+    return readFile(binaryCacheDir + "/" + path);
+}
+
+}
diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh
new file mode 100644
index 00000000..91c56abc
--- /dev/null
+++ b/src/hydra-queue-runner/local-binary-cache-store.hh
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "binary-cache-store.hh"
+
+namespace nix {
+
+class LocalBinaryCacheStore : public BinaryCacheStore
+{
+private:
+
+    Path binaryCacheDir;
+
+public:
+
+    LocalBinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
+        const Path & secretKeyFile, const Path & publicKeyFile);
+
+    void init() override;
+
+protected:
+
+    bool fileExists(const std::string & path) override;
+
+    void upsertFile(const std::string & path, const std::string & data) override;
+
+    std::string getFile(const std::string & path) override;
+
+};
+
+}

From 2d40888e2e9a747dbaf32cc3b956454b17dd1bb4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 16:18:50 +0100
Subject: [PATCH 07/40] Add an S3-backed binary cache store

---
 release.nix                                   |   4 +
 src/hydra-queue-runner/Makefile.am            |   5 +-
 src/hydra-queue-runner/binary-cache-store.cc  |  12 +-
 src/hydra-queue-runner/hydra-queue-runner.cc  |  11 +-
 .../local-binary-cache-store.cc               |   3 +-
 .../local-binary-cache-store.hh               |   5 +-
 .../s3-binary-cache-store.cc                  | 134 ++++++++++++++++++
 .../s3-binary-cache-store.hh                  |  41 ++++++
 8 files changed, 205 insertions(+), 10 deletions(-)
 create mode 100644 src/hydra-queue-runner/s3-binary-cache-store.cc
 create mode 100644 src/hydra-queue-runner/s3-binary-cache-store.hh

diff --git a/release.nix b/release.nix
index 70f3505b..e482b2a1 100644
--- a/release.nix
+++ b/release.nix
@@ -159,6 +159,10 @@ rec {
           guile # optional, for Guile + Guix support
           perlDeps perl
           postgresql92 # for running the tests
+          (aws-sdk-cpp.override {
+            apis = ["s3"];
+            customMemoryManagement = false;
+          })
         ];
 
       hydraPath = lib.makeSearchPath "bin" (
diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 3b5b48a2..6d526c89 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -4,7 +4,8 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c
  builder.cc build-result.cc build-remote.cc  \
  build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
  binary-cache-store.hh binary-cache-store.cc \
- local-binary-cache-store.hh local-binary-cache-store.cc
+ local-binary-cache-store.hh local-binary-cache-store.cc \
+ s3-binary-cache-store.hh s3-binary-cache-store.cc
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
-AM_CXXFLAGS = $(NIX_CFLAGS) -Wall
+AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3
diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index 80720ba0..d9cd0a4e 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -7,6 +7,8 @@
 #include "nar-info.hh"
 #include "worker-protocol.hh"
 
+#include <chrono>
+
 namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(ref<Store> localStore,
@@ -50,15 +52,19 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     if (info.narHash.type != htUnknown && info.narHash != narInfo.narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
-    printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
-        % info.path % info.narSize);
-
     /* Compress the NAR. */
     narInfo.compression = "xz";
+    auto now1 = std::chrono::steady_clock::now();
     string narXz = compressXZ(nar);
+    auto now2 = std::chrono::steady_clock::now();
     narInfo.fileHash = hashString(htSHA256, narXz);
     narInfo.fileSize = narXz.size();
 
+    printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
+        % info.path % info.narSize
+        % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
+        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+
     /* Atomically write the NAR file. */
     narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
     if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 8f363ef3..d95f9e94 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -8,6 +8,7 @@
 #include "state.hh"
 #include "build-result.hh"
 #include "local-binary-cache-store.hh"
+#include "s3-binary-cache-store.hh"
 
 #include "shared.hh"
 #include "globals.hh"
@@ -33,10 +34,16 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
+#if 0
     auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
-        "/tmp/binary-cache",
         "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public");
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "/tmp/binary-cache");
+#endif
+    auto store = make_ref<S3BinaryCacheStore>(getLocalStore(),
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "nix-test-cache-3");
     store->init();
     return store;
 }
diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc
index f0b2637f..250eb3c1 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.cc
+++ b/src/hydra-queue-runner/local-binary-cache-store.cc
@@ -3,7 +3,8 @@
 namespace nix {
 
 LocalBinaryCacheStore::LocalBinaryCacheStore(ref<Store> localStore,
-    const Path & binaryCacheDir,  const Path & secretKeyFile, const Path & publicKeyFile)
+    const Path & secretKeyFile, const Path & publicKeyFile,
+    const Path & binaryCacheDir)
     : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
     , binaryCacheDir(binaryCacheDir)
 {
diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh
index 91c56abc..26bac146 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.hh
+++ b/src/hydra-queue-runner/local-binary-cache-store.hh
@@ -12,8 +12,9 @@ private:
 
 public:
 
-    LocalBinaryCacheStore(ref<Store> localStore, const Path & binaryCacheDir,
-        const Path & secretKeyFile, const Path & publicKeyFile);
+    LocalBinaryCacheStore(ref<Store> localStore,
+        const Path & secretKeyFile, const Path & publicKeyFile,
+        const Path & binaryCacheDir);
 
     void init() override;
 
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
new file mode 100644
index 00000000..c00cf9a9
--- /dev/null
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -0,0 +1,134 @@
+#include "s3-binary-cache-store.hh"
+
+#include <aws/core/client/ClientConfiguration.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/CreateBucketRequest.h>
+#include <aws/s3/model/GetBucketLocationRequest.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+
+namespace nix {
+
+/* Helper: given an Outcome<R, E>, return R in case of success, or
+   throw an exception in case of an error. */
+template<typename R, typename E>
+R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
+{
+    if (!outcome.IsSuccess())
+        throw Error(format("AWS error: %1%") % outcome.GetError().GetMessage());
+    return outcome.GetResultWithOwnership();
+}
+
+S3BinaryCacheStore::S3BinaryCacheStore(ref<Store> localStore,
+    const Path & secretKeyFile, const Path & publicKeyFile,
+    const std::string & bucketName)
+    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    , bucketName(bucketName)
+    , config(makeConfig())
+    , client(make_ref<Aws::S3::S3Client>(*config))
+{
+}
+
+ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()
+{
+    auto res = make_ref<Aws::Client::ClientConfiguration>();
+    res->region = Aws::Region::EU_WEST_1;
+    res->requestTimeoutMs = 600 * 1000;
+    return res;
+}
+
+void S3BinaryCacheStore::init()
+{
+    /* Create the bucket if it doesn't already exists. */
+    // FIXME: HeadBucket would be more appropriate, but doesn't return
+    // an easily parsed 404 message.
+    auto res = client->GetBucketLocation(
+        Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
+
+    if (!res.IsSuccess()) {
+        if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
+            throw Error(format("AWS error: %1%") % res.GetError().GetMessage());
+
+        checkAws(client->CreateBucket(
+            Aws::S3::Model::CreateBucketRequest()
+            .WithBucket(bucketName)
+            .WithCreateBucketConfiguration(
+                Aws::S3::Model::CreateBucketConfiguration()
+                .WithLocationConstraint(
+                    Aws::S3::Model::BucketLocationConstraint::eu_west_1))));
+    }
+
+    BinaryCacheStore::init();
+}
+
+bool S3BinaryCacheStore::fileExists(const std::string & path)
+{
+    auto res = client->HeadObject(
+        Aws::S3::Model::HeadObjectRequest()
+        .WithBucket(bucketName)
+        .WithKey(path));
+
+    if (!res.IsSuccess()) {
+        auto & error = res.GetError();
+        if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
+            && error.GetMessage().find("404") != std::string::npos)
+            return false;
+        throw Error(format("AWS error: %1%") % error.GetMessage());
+    }
+
+    return true;
+}
+
+void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
+{
+    auto request =
+        Aws::S3::Model::PutObjectRequest()
+        .WithBucket(bucketName)
+        .WithKey(path);
+
+    auto stream = std::make_shared<std::stringstream>(data);
+
+    request.SetBody(stream);
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    auto result = checkAws(client->PutObject(request));
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    printMsg(lvlError, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
+        % bucketName % path
+        % data.size()
+        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+}
+
+std::string S3BinaryCacheStore::getFile(const std::string & path)
+{
+    auto request =
+        Aws::S3::Model::GetObjectRequest()
+        .WithBucket(bucketName)
+        .WithKey(path);
+
+    request.SetResponseStreamFactory([&]() {
+        return Aws::New<std::stringstream>("STRINGSTREAM");
+    });
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    auto result = checkAws(client->GetObject(request));
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();
+
+    printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3%) in %4% ms")
+        % bucketName % path
+        % res.size()
+        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+
+    return res;
+}
+
+}
+
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh
new file mode 100644
index 00000000..2c11a164
--- /dev/null
+++ b/src/hydra-queue-runner/s3-binary-cache-store.hh
@@ -0,0 +1,41 @@
+#pragma once
+
+#include "binary-cache-store.hh"
+
+namespace Aws { namespace Client { class ClientConfiguration; } }
+namespace Aws { namespace S3 { class S3Client; } }
+
+namespace nix {
+
+class S3BinaryCacheStore : public BinaryCacheStore
+{
+private:
+
+    std::string bucketName;
+
+    ref<Aws::Client::ClientConfiguration> config;
+    ref<Aws::S3::S3Client> client;
+
+public:
+
+    S3BinaryCacheStore(ref<Store> localStore,
+        const Path & secretKeyFile, const Path & publicKeyFile,
+        const std::string & bucketName);
+
+    void init() override;
+
+private:
+
+    ref<Aws::Client::ClientConfiguration> makeConfig();
+
+protected:
+
+    bool fileExists(const std::string & path) override;
+
+    void upsertFile(const std::string & path, const std::string & data) override;
+
+    std::string getFile(const std::string & path) override;
+
+};
+
+}

From db3fcc0f5ef12b0215ba95a126d347c756e74fe0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 16:42:05 +0100
Subject: [PATCH 08/40] Enable substitution on the build machines

If properly configured, this allows them to get store paths directly
from S3, rather than having to receive them from the queue runner.
---
 src/hydra-queue-runner/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index f18df06a..e6be7b10 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -238,7 +238,7 @@ void State::buildRemote(ref<Store> destStore,
 
         auto now1 = std::chrono::steady_clock::now();
 
-        copyClosureTo(destStore, from, to, inputs, bytesSent);
+        copyClosureTo(destStore, from, to, inputs, bytesSent, true);
 
         auto now2 = std::chrono::steady_clock::now();
 

From 8c9fc677c1bb40dae576849f7d35c10553ea549f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 16:43:24 +0100
Subject: [PATCH 09/40] Typo

---
 src/hydra-queue-runner/s3-binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index c00cf9a9..0fa33b25 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -122,7 +122,7 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
 
     auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();
 
-    printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3%) in %4% ms")
+    printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
         % bucketName % path
         % res.size()
         % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());

From 00a7be13a2e5fc73937363749bb00b34d4cf6556 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 17:11:46 +0100
Subject: [PATCH 10/40] Make queue runner internal status available under
 /queue-runner-status

---
 src/hydra-queue-runner/hydra-queue-runner.cc |  4 ++--
 src/lib/Hydra/Controller/Root.pm             | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index d95f9e94..d64ffd0c 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -705,10 +705,10 @@ void State::run(BuildID buildOne)
     while (true) {
         try {
             auto conn(dbPool.get());
-            receiver dumpStatus(*conn, "dump_status");
+            receiver dumpStatus_(*conn, "dump_status");
             while (true) {
                 bool timeout = conn->await_notification(300, 0) == 0;
-                State::dumpStatus(*conn, timeout);
+                dumpStatus(*conn, timeout);
             }
         } catch (std::exception & e) {
             printMsg(lvlError, format("main thread: %1%") % e.what());
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index dab82b62..48319479 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -118,6 +118,22 @@ sub status_GET {
 }
 
 
+sub queue_runner_status :Local :Path('queue-runner-status') :Args(0) :ActionClass('REST') { }
+
+sub queue_runner_status_GET {
+    my ($self, $c) = @_;
+
+    #my $status = from_json($c->model('DB::SystemStatus')->find('queue-runner')->status);
+    my $status = from_json(`hydra-queue-runner --status`);
+    if ($?) { $status->{status} = "unknown"; }
+    my $json = JSON->new->pretty()->canonical();
+
+    $c->stash->{template} = 'queue-runner-status.tt';
+    $c->stash->{status} = $json->encode($status);
+    $self->status_ok($c, entity => $status);
+}
+
+
 sub machines :Local Args(0) {
     my ($self, $c) = @_;
     my $machines = getMachines;

From dc4a00347db3ab1610710fba7fa12e206190c4bc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 18 Feb 2016 17:31:19 +0100
Subject: [PATCH 11/40] Use a single BinaryCacheStore for all threads

This will make it easier to do caching / keep stats. Also, we won't
have S3Client's connection pooling if we create multiple S3Client
instances.
---
 src/hydra-queue-runner/binary-cache-store.cc  |  8 ++++--
 src/hydra-queue-runner/binary-cache-store.hh  | 11 ++++++--
 src/hydra-queue-runner/hydra-queue-runner.cc  | 28 +++++++++++--------
 .../local-binary-cache-store.cc               |  4 +--
 .../local-binary-cache-store.hh               |  2 +-
 .../s3-binary-cache-store.cc                  |  4 +--
 .../s3-binary-cache-store.hh                  |  2 +-
 src/hydra-queue-runner/state.hh               |  3 ++
 8 files changed, 39 insertions(+), 23 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index d9cd0a4e..bf9215d5 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -11,9 +11,9 @@
 
 namespace nix {
 
-BinaryCacheStore::BinaryCacheStore(ref<Store> localStore,
+BinaryCacheStore::BinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile)
-    : localStore(localStore)
+    : storeFactory(storeFactory)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@@ -192,6 +192,8 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     PathSet left;
 
+    auto localStore = storeFactory();
+
     for (auto & storePath : paths) {
         if (!localStore->isValidPath(storePath)) {
             left.insert(storePath);
@@ -210,6 +212,8 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
 
 void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
+    auto localStore = storeFactory();
+
     for (auto & storePath : paths) {
         assert(!isDerivation(storePath));
 
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 0db7ddc4..8883075b 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -7,18 +7,23 @@ namespace nix {
 
 struct NarInfo;
 
+/* While BinaryCacheStore is thread-safe, LocalStore and RemoteStore
+   aren't. Until they are, use a factory to produce a thread-local
+   local store. */
+typedef std::function<ref<Store>()> StoreFactory;
+
 class BinaryCacheStore : public Store
 {
 private:
 
-    ref<Store> localStore;
-
     std::unique_ptr<SecretKey> secretKey;
     std::unique_ptr<PublicKeys> publicKeys;
 
+    StoreFactory storeFactory;
+
 protected:
 
-    BinaryCacheStore(ref<Store> localStore,
+    BinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile);
 
     virtual bool fileExists(const std::string & path) = 0;
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index d64ffd0c..afa4a565 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -23,6 +23,21 @@ State::State()
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
 
     logDir = canonPath(hydraData + "/build-logs");
+
+#if 0
+    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "/tmp/binary-cache");
+#endif
+
+    auto store = std::make_shared<S3BinaryCacheStore>(
+        []() { return openStore(); },
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "nix-test-cache-3");;
+    store->init();
+    _destStore = store;
 }
 
 
@@ -34,18 +49,7 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
-#if 0
-    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "/tmp/binary-cache");
-#endif
-    auto store = make_ref<S3BinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "nix-test-cache-3");
-    store->init();
-    return store;
+    return ref<Store>(_destStore);
 }
 
 
diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc
index 250eb3c1..a82e6597 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.cc
+++ b/src/hydra-queue-runner/local-binary-cache-store.cc
@@ -2,10 +2,10 @@
 
 namespace nix {
 
-LocalBinaryCacheStore::LocalBinaryCacheStore(ref<Store> localStore,
+LocalBinaryCacheStore::LocalBinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const Path & binaryCacheDir)
-    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
     , binaryCacheDir(binaryCacheDir)
 {
 }
diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh
index 26bac146..e29d0ca2 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.hh
+++ b/src/hydra-queue-runner/local-binary-cache-store.hh
@@ -12,7 +12,7 @@ private:
 
 public:
 
-    LocalBinaryCacheStore(ref<Store> localStore,
+    LocalBinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const Path & binaryCacheDir);
 
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 0fa33b25..9d545d29 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -20,10 +20,10 @@ R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
     return outcome.GetResultWithOwnership();
 }
 
-S3BinaryCacheStore::S3BinaryCacheStore(ref<Store> localStore,
+S3BinaryCacheStore::S3BinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const std::string & bucketName)
-    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
     , bucketName(bucketName)
     , config(makeConfig())
     , client(make_ref<Aws::S3::S3Client>(*config))
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh
index 2c11a164..a39a498b 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.hh
+++ b/src/hydra-queue-runner/s3-binary-cache-store.hh
@@ -18,7 +18,7 @@ private:
 
 public:
 
-    S3BinaryCacheStore(ref<Store> localStore,
+    S3BinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const std::string & bucketName);
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index aa9c1f38..eb9d6457 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -346,6 +346,9 @@ private:
 
     std::atomic<time_t> lastDispatcherCheck{0};
 
+    /* Destination store. */
+    std::shared_ptr<nix::Store> _destStore;
+
 public:
     State();
 

From a0f74047da1e49a375b6556ebfe89636b0d717ca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 19 Feb 2016 14:24:23 +0100
Subject: [PATCH 12/40] Keep some statistics for the binary cache stores

---
 src/hydra-queue-runner/binary-cache-store.cc  | 27 +++++-
 src/hydra-queue-runner/binary-cache-store.hh  | 20 ++++
 src/hydra-queue-runner/hydra-queue-runner.cc  | 97 ++++++++++++++-----
 .../s3-binary-cache-store.cc                  | 29 ++++--
 .../s3-binary-cache-store.hh                  | 17 ++++
 5 files changed, 160 insertions(+), 30 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index bf9215d5..db25b7c6 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -32,6 +32,11 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
 }
 
+const BinaryCacheStore::Stats & BinaryCacheStore::getStats()
+{
+    return stats;
+}
+
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
 {
     assertStorePath(storePath);
@@ -60,23 +65,36 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     narInfo.fileHash = hashString(htSHA256, narXz);
     narInfo.fileSize = narXz.size();
 
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % info.path % info.narSize
         % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
-        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+        % duration);
 
     /* Atomically write the NAR file. */
     narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
-    if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz);
+    if (!fileExists(narInfo.url)) {
+        stats.narWrite++;
+        upsertFile(narInfo.url, narXz);
+    } else
+        stats.narWriteAverted++;
+
+    stats.narWriteBytes += nar.size();
+    stats.narWriteCompressedBytes += narXz.size();
+    stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo.sign(*secretKey);
 
     upsertFile(narInfoFile, narInfo.to_string());
+
+    stats.narInfoWrite++;
 }
 
 NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 {
+    stats.narInfoRead++;
+
     auto narInfoFile = narInfoFileFor(storePath);
     auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile);
     assert(narInfo.path == storePath);
@@ -102,6 +120,9 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
 
     auto nar = getFile(res.url);
 
+    stats.narRead++;
+    stats.narReadCompressedBytes += nar.size();
+
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
     if (res.compression == "none")
@@ -111,6 +132,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
     else
         throw Error(format("unknown NAR compression type ‘%1%’") % nar);
 
+    stats.narReadBytes += nar.size();
+
     printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
 
     assert(nar.size() % 8 == 0);
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 8883075b..019b2611 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -3,6 +3,8 @@
 #include "crypto.hh"
 #include "store-api.hh"
 
+#include <atomic>
+
 namespace nix {
 
 struct NarInfo;
@@ -36,8 +38,26 @@ public:
 
     virtual void init();
 
+    struct Stats
+    {
+        std::atomic<uint64_t> narInfoRead{0};
+        std::atomic<uint64_t> narInfoWrite{0};
+        std::atomic<uint64_t> narRead{0};
+        std::atomic<uint64_t> narReadBytes{0};
+        std::atomic<uint64_t> narReadCompressedBytes{0};
+        std::atomic<uint64_t> narWrite{0};
+        std::atomic<uint64_t> narWriteAverted{0};
+        std::atomic<uint64_t> narWriteBytes{0};
+        std::atomic<uint64_t> narWriteCompressedBytes{0};
+        std::atomic<uint64_t> narWriteCompressionTimeMs{0};
+    };
+
+    const Stats & getStats();
+
 private:
 
+    Stats stats;
+
     std::string narInfoFileFor(const Path & storePath);
 
     void addToCache(const ValidPathInfo & info, const string & nar);
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index afa4a565..48c6865a 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -23,21 +23,6 @@ State::State()
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
 
     logDir = canonPath(hydraData + "/build-logs");
-
-#if 0
-    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "/tmp/binary-cache");
-#endif
-
-    auto store = std::make_shared<S3BinaryCacheStore>(
-        []() { return openStore(); },
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "nix-test-cache-3");;
-    store->init();
-    _destStore = store;
 }
 
 
@@ -532,8 +517,8 @@ void State::dumpStatus(Connection & conn, bool log)
         root.attr("nrStepsCopyingTo", nrStepsCopyingTo);
         root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom);
         root.attr("nrStepsWaiting", nrStepsWaiting);
-        root.attr("bytesSent"); out << bytesSent;
-        root.attr("bytesReceived"); out << bytesReceived;
+        root.attr("bytesSent", bytesSent);
+        root.attr("bytesReceived", bytesReceived);
         root.attr("nrBuildsRead", nrBuildsRead);
         root.attr("nrBuildsDone", nrBuildsDone);
         root.attr("nrStepsDone", nrStepsDone);
@@ -542,8 +527,8 @@ void State::dumpStatus(Connection & conn, bool log)
         if (nrStepsDone) {
             root.attr("totalStepTime", totalStepTime);
             root.attr("totalStepBuildTime", totalStepBuildTime);
-            root.attr("avgStepTime"); out << (float) totalStepTime / nrStepsDone;
-            root.attr("avgStepBuildTime"); out << (float) totalStepBuildTime / nrStepsDone;
+            root.attr("avgStepTime", (float) totalStepTime / nrStepsDone);
+            root.attr("avgStepBuildTime", (float) totalStepBuildTime / nrStepsDone);
         }
         root.attr("nrQueueWakeups", nrQueueWakeups);
         root.attr("nrDispatcherWakeups", nrDispatcherWakeups);
@@ -565,8 +550,8 @@ void State::dumpStatus(Connection & conn, bool log)
                 if (m->state->nrStepsDone) {
                     nested2.attr("totalStepTime", s->totalStepTime);
                     nested2.attr("totalStepBuildTime", s->totalStepBuildTime);
-                    nested2.attr("avgStepTime"); out << (float) s->totalStepTime / s->nrStepsDone;
-                    nested2.attr("avgStepBuildTime"); out << (float) s->totalStepBuildTime / s->nrStepsDone;
+                    nested2.attr("avgStepTime", (float) s->totalStepTime / s->nrStepsDone);
+                    nested2.attr("avgStepBuildTime", (float) s->totalStepBuildTime / s->nrStepsDone);
                 }
             }
         }
@@ -577,7 +562,7 @@ void State::dumpStatus(Connection & conn, bool log)
             for (auto & jobset : *jobsets_) {
                 nested.attr(jobset.first.first + ":" + jobset.first.second);
                 JSONObject nested2(out);
-                nested2.attr("shareUsed"); out << jobset.second->shareUsed();
+                nested2.attr("shareUsed", jobset.second->shareUsed());
                 nested2.attr("seconds", jobset.second->getSeconds());
             }
         }
@@ -597,6 +582,59 @@ void State::dumpStatus(Connection & conn, bool log)
                     nested2.attr("lastActive", std::chrono::system_clock::to_time_t(i.second.lastActive));
             }
         }
+
+        auto store = dynamic_cast<S3BinaryCacheStore *>(&*getDestStore());
+
+        if (store) {
+            root.attr("store");
+            JSONObject nested(out);
+
+            auto & stats = store->getStats();
+            nested.attr("narInfoRead", stats.narInfoRead);
+            nested.attr("narInfoWrite", stats.narInfoWrite);
+            nested.attr("narRead", stats.narRead);
+            nested.attr("narReadBytes", stats.narReadBytes);
+            nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
+            nested.attr("narWrite", stats.narWrite);
+            nested.attr("narWriteAverted", stats.narWriteAverted);
+            nested.attr("narWriteBytes", stats.narWriteBytes);
+            nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);
+            nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);
+            nested.attr("narCompressionSavings",
+                stats.narWriteBytes
+                ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
+                : 0.0);
+            nested.attr("narCompressionSpeed", // MiB/s
+                stats.narWriteCompressionTimeMs
+                ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
+                : 0.0);
+
+            auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
+            if (s3Store) {
+                nested.attr("s3");
+                JSONObject nested2(out);
+                auto & s3Stats = s3Store->getS3Stats();
+                nested2.attr("put", s3Stats.put);
+                nested2.attr("putBytes", s3Stats.putBytes);
+                nested2.attr("putTimeMs", s3Stats.putTimeMs);
+                nested2.attr("putSpeed",
+                    s3Stats.putTimeMs
+                    ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
+                    : 0.0);
+                nested2.attr("get", s3Stats.get);
+                nested2.attr("getBytes", s3Stats.getBytes);
+                nested2.attr("getTimeMs", s3Stats.getTimeMs);
+                nested2.attr("getSpeed",
+                    s3Stats.getTimeMs
+                    ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
+                    : 0.0);
+                nested2.attr("head", s3Stats.head);
+                nested2.attr("costDollarApprox",
+                    (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
+                    + s3Stats.put / 1000.0 * 0.005 +
+                    + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
+            }
+        }
     }
 
     if (log) printMsg(lvlInfo, format("status: %1%") % out.str());
@@ -685,6 +723,21 @@ void State::run(BuildID buildOne)
     if (!lock)
         throw Error("hydra-queue-runner is already running");
 
+#if 0
+    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "/tmp/binary-cache");
+#endif
+
+    auto store = std::make_shared<S3BinaryCacheStore>(
+        []() { return openStore(); },
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "nix-test-cache-3");;
+    store->init();
+    _destStore = store;
+
     {
         auto conn(dbPool.get());
         clearBusy(*conn, 0);
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 9d545d29..cec6f9cf 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -62,8 +62,15 @@ void S3BinaryCacheStore::init()
     BinaryCacheStore::init();
 }
 
+const S3BinaryCacheStore::Stats & S3BinaryCacheStore::getS3Stats()
+{
+    return stats;
+}
+
 bool S3BinaryCacheStore::fileExists(const std::string & path)
 {
+    stats.head++;
+
     auto res = client->HeadObject(
         Aws::S3::Model::HeadObjectRequest()
         .WithBucket(bucketName)
@@ -91,16 +98,21 @@ void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string
 
     request.SetBody(stream);
 
+    stats.put++;
+    stats.putBytes += data.size();
+
     auto now1 = std::chrono::steady_clock::now();
 
     auto result = checkAws(client->PutObject(request));
 
     auto now2 = std::chrono::steady_clock::now();
 
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+
     printMsg(lvlError, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
-        % bucketName % path
-        % data.size()
-        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+        % bucketName % path % data.size() % duration);
+
+    stats.putTimeMs += duration;
 }
 
 std::string S3BinaryCacheStore::getFile(const std::string & path)
@@ -114,6 +126,8 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
+    stats.get++;
+
     auto now1 = std::chrono::steady_clock::now();
 
     auto result = checkAws(client->GetObject(request));
@@ -122,10 +136,13 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
 
     auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();
 
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+
     printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
-        % bucketName % path
-        % res.size()
-        % std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count());
+        % bucketName % path % res.size() % duration);
+
+    stats.getBytes += res.size();
+    stats.getTimeMs += duration;
 
     return res;
 }
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh
index a39a498b..15d717c9 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.hh
+++ b/src/hydra-queue-runner/s3-binary-cache-store.hh
@@ -2,6 +2,8 @@
 
 #include "binary-cache-store.hh"
 
+#include <atomic>
+
 namespace Aws { namespace Client { class ClientConfiguration; } }
 namespace Aws { namespace S3 { class S3Client; } }
 
@@ -24,8 +26,23 @@ public:
 
     void init() override;
 
+    struct Stats
+    {
+        std::atomic<uint64_t> put{0};
+        std::atomic<uint64_t> putBytes{0};
+        std::atomic<uint64_t> putTimeMs{0};
+        std::atomic<uint64_t> get{0};
+        std::atomic<uint64_t> getBytes{0};
+        std::atomic<uint64_t> getTimeMs{0};
+        std::atomic<uint64_t> head{0};
+    };
+
+    const Stats & getS3Stats();
+
 private:
 
+    Stats stats;
+
     ref<Aws::Client::ClientConfiguration> makeConfig();
 
 protected:

From bd76f9120a1c600e65fcfcc9ba3d98c0e91cc80a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 19 Feb 2016 16:19:40 +0100
Subject: [PATCH 13/40] Cache .narinfo lookups

---
 src/hydra-queue-runner/binary-cache-store.cc | 24 +++++-
 src/hydra-queue-runner/binary-cache-store.hh | 12 +++
 src/hydra-queue-runner/hydra-queue-runner.cc |  2 +
 src/hydra-queue-runner/lru-cache.hh          | 80 ++++++++++++++++++++
 src/hydra-queue-runner/sync.hh               |  2 +-
 5 files changed, 115 insertions(+), 5 deletions(-)
 create mode 100644 src/hydra-queue-runner/lru-cache.hh

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index db25b7c6..226c8dda 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -1,4 +1,5 @@
 #include "binary-cache-store.hh"
+#include "sync.hh"
 
 #include "archive.hh"
 #include "compression.hh"
@@ -93,18 +94,33 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
 
 NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
 {
+    {
+        auto state_(state.lock());
+        auto res = state_->narInfoCache.get(storePath);
+        if (res) {
+            stats.narInfoReadAverted++;
+            return **res;
+        }
+    }
+
     stats.narInfoRead++;
 
     auto narInfoFile = narInfoFileFor(storePath);
-    auto narInfo = NarInfo(getFile(narInfoFile), narInfoFile);
-    assert(narInfo.path == storePath);
+    auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
+    assert(narInfo->path == storePath);
 
     if (publicKeys) {
-        if (!narInfo.checkSignature(*publicKeys))
+        if (!narInfo->checkSignature(*publicKeys))
             throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
     }
 
-    return narInfo;
+    {
+        auto state_(state.lock());
+        state_->narInfoCache.upsert(storePath, narInfo);
+        stats.narInfoCacheSize = state_->narInfoCache.size();
+    }
+
+    return *narInfo;
 }
 
 bool BinaryCacheStore::isValidPath(const Path & storePath)
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 019b2611..709a18cc 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -3,6 +3,9 @@
 #include "crypto.hh"
 #include "store-api.hh"
 
+#include "lru-cache.hh"
+#include "sync.hh"
+
 #include <atomic>
 
 namespace nix {
@@ -23,6 +26,13 @@ private:
 
     StoreFactory storeFactory;
 
+    struct State
+    {
+        LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024};
+    };
+
+    Sync<State> state;
+
 protected:
 
     BinaryCacheStore(const StoreFactory & storeFactory,
@@ -41,7 +51,9 @@ public:
     struct Stats
     {
         std::atomic<uint64_t> narInfoRead{0};
+        std::atomic<uint64_t> narInfoReadAverted{0};
         std::atomic<uint64_t> narInfoWrite{0};
+        std::atomic<uint64_t> narInfoCacheSize{0};
         std::atomic<uint64_t> narRead{0};
         std::atomic<uint64_t> narReadBytes{0};
         std::atomic<uint64_t> narReadCompressedBytes{0};
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 48c6865a..8080af7a 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -591,7 +591,9 @@ void State::dumpStatus(Connection & conn, bool log)
 
             auto & stats = store->getStats();
             nested.attr("narInfoRead", stats.narInfoRead);
+            nested.attr("narInfoReadAverted", stats.narInfoReadAverted);
             nested.attr("narInfoWrite", stats.narInfoWrite);
+            nested.attr("narInfoCacheSize", stats.narInfoCacheSize);
             nested.attr("narRead", stats.narRead);
             nested.attr("narReadBytes", stats.narReadBytes);
             nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
diff --git a/src/hydra-queue-runner/lru-cache.hh b/src/hydra-queue-runner/lru-cache.hh
new file mode 100644
index 00000000..113411bd
--- /dev/null
+++ b/src/hydra-queue-runner/lru-cache.hh
@@ -0,0 +1,80 @@
+#pragma once
+
+#include <map>
+#include <list>
+
+/* A simple least-recently used cache. Not thread-safe. */
+template<typename Key, typename Value>
+class LRUCache
+{
+private:
+
+    size_t maxSize;
+
+    // Stupid wrapper to get around circular dependency between Data
+    // and LRU.
+    struct LRUIterator;
+
+    using Data = std::map<Key, std::pair<LRUIterator, Value>>;
+    using LRU = std::list<typename Data::iterator>;
+
+    struct LRUIterator { typename LRU::iterator it; };
+
+    Data data;
+    LRU lru;
+
+public:
+
+    LRUCache(size_t maxSize) : maxSize(maxSize) { }
+
+    /* Insert or upsert an item in the cache. */
+    void upsert(const Key & key, const Value & value)
+    {
+        erase(key);
+
+        if (data.size() >= maxSize) {
+            /* Retire the oldest item. */
+            auto oldest = lru.begin();
+            data.erase(*oldest);
+            lru.erase(oldest);
+        }
+
+        auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
+        assert(res.second);
+        auto & i(res.first);
+
+        auto j = lru.insert(lru.end(), i);
+
+        i->second.first.it = j;
+    }
+
+    bool erase(const Key & key)
+    {
+        auto i = data.find(key);
+        if (i == data.end()) return false;
+        lru.erase(i->second.first.it);
+        data.erase(i);
+        return true;
+    }
+
+    /* Look up an item in the cache. If it's exists, it becomes the
+       most recently used item. */
+    // FIXME: use boost::optional?
+    Value * get(const Key & key)
+    {
+        auto i = data.find(key);
+        if (i == data.end()) return 0;
+
+        /* Move this item to the back of the LRU list. */
+        lru.erase(i->second.first.it);
+        auto j = lru.insert(lru.end(), i);
+        i->second.first.it = j;
+
+        return &i->second.second;
+    }
+
+    size_t size()
+    {
+        return data.size();
+    }
+};
diff --git a/src/hydra-queue-runner/sync.hh b/src/hydra-queue-runner/sync.hh
index 1573f091..4d53a6ee 100644
--- a/src/hydra-queue-runner/sync.hh
+++ b/src/hydra-queue-runner/sync.hh
@@ -20,7 +20,7 @@
    scope.
 */
 
-template <class T>
+template<class T>
 class Sync
 {
 private:

From 2b76094a237cee253ebfc89bab448a5ff2e503b8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 19 Feb 2016 17:41:11 +0100
Subject: [PATCH 14/40] S3BinaryCacheStore::isValidPath(): Do a GET instead of
 HEAD

---
 src/hydra-queue-runner/binary-cache-store.cc  |  4 +--
 src/hydra-queue-runner/binary-cache-store.hh  |  2 ++
 .../s3-binary-cache-store.cc                  | 28 ++++++++++++++++++-
 .../s3-binary-cache-store.hh                  |  2 ++
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index 226c8dda..6370c3b4 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -103,12 +103,12 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
         }
     }
 
-    stats.narInfoRead++;
-
     auto narInfoFile = narInfoFileFor(storePath);
     auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
     assert(narInfo->path == storePath);
 
+    stats.narInfoRead++;
+
     if (publicKeys) {
         if (!narInfo->checkSignature(*publicKeys))
             throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 709a18cc..8b2d4a81 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -74,6 +74,8 @@ private:
 
     void addToCache(const ValidPathInfo & info, const string & nar);
 
+protected:
+
     NarInfo readNarInfo(const Path & storePath);
 
 public:
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index cec6f9cf..3f6b6bf9 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -1,5 +1,7 @@
 #include "s3-binary-cache-store.hh"
 
+#include "nar-info.hh"
+
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
@@ -10,13 +12,22 @@
 
 namespace nix {
 
+struct S3Error : public Error
+{
+    Aws::S3::S3Errors err;
+    S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
+        : Error(fs), err(err) { };
+};
+
 /* Helper: given an Outcome<R, E>, return R in case of success, or
    throw an exception in case of an error. */
 template<typename R, typename E>
 R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
 {
     if (!outcome.IsSuccess())
-        throw Error(format("AWS error: %1%") % outcome.GetError().GetMessage());
+        throw S3Error(
+            outcome.GetError().GetErrorType(),
+            format("AWS error: %1%") % outcome.GetError().GetMessage());
     return outcome.GetResultWithOwnership();
 }
 
@@ -67,6 +78,21 @@ const S3BinaryCacheStore::Stats & S3BinaryCacheStore::getS3Stats()
     return stats;
 }
 
+/* This is a specialisation of isValidPath() that optimistically
+   fetches the .narinfo file, rather than first checking for its
+   existence via a HEAD request. Since .narinfos are small, doing a
+   GET is unlikely to be slower than HEAD. */
+bool S3BinaryCacheStore::isValidPath(const Path & storePath)
+{
+    try {
+        readNarInfo(storePath);
+        return true;
+    } catch (S3Error & e) {
+        if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return false;
+        throw;
+    }
+}
+
 bool S3BinaryCacheStore::fileExists(const std::string & path)
 {
     stats.head++;
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh
index 15d717c9..a0dd7813 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.hh
+++ b/src/hydra-queue-runner/s3-binary-cache-store.hh
@@ -39,6 +39,8 @@ public:
 
     const Stats & getS3Stats();
 
+    bool isValidPath(const Path & storePath) override;
+
 private:
 
     Stats stats;

From a593ebc58e8c77402365c88354ea83499f93704b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 19 Feb 2016 20:58:40 +0100
Subject: [PATCH 15/40] Add missing file

---
 src/root/queue-runner-status.tt | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 src/root/queue-runner-status.tt

diff --git a/src/root/queue-runner-status.tt b/src/root/queue-runner-status.tt
new file mode 100644
index 00000000..d2896973
--- /dev/null
+++ b/src/root/queue-runner-status.tt
@@ -0,0 +1,8 @@
+[% WRAPPER layout.tt title="Queue runner status" %]
+[% PROCESS common.tt %]
+
+<pre>
+[% HTML.escape(status) %]
+</pre>
+
+[% END %]

From 1cefd6cac851b75f8110a63b128d1554c2fffa29 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Sat, 20 Feb 2016 00:02:37 +0100
Subject: [PATCH 16/40] Fix log message

---
 src/hydra-queue-runner/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index 6370c3b4..873b83b4 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -68,7 +68,7 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
-        % info.path % info.narSize
+        % narInfo.path % narInfo.narSize
         % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
         % duration);
 

From 88a05763cc444d990c4aa0c78d9a93772fa421e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Sat, 20 Feb 2016 00:04:08 +0100
Subject: [PATCH 17/40] Pool local store connections

---
 src/hydra-queue-runner/binary-cache-store.cc | 14 +++++++-------
 src/hydra-queue-runner/binary-cache-store.hh |  4 +++-
 src/hydra-queue-runner/hydra-queue-runner.cc | 14 ++++++++------
 src/hydra-queue-runner/pool.hh               | 14 +++++++++++++-
 src/hydra-queue-runner/queue-monitor.cc      |  2 +-
 src/hydra-queue-runner/state.hh              |  7 ++++++-
 6 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index 873b83b4..f4e9b652 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -234,11 +234,11 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
     auto localStore = storeFactory();
 
     for (auto & storePath : paths) {
-        if (!localStore->isValidPath(storePath)) {
+        if (!(*localStore)->isValidPath(storePath)) {
             left.insert(storePath);
             continue;
         }
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
+        ValidPathInfo info = (*localStore)->queryPathInfo(storePath);
         SubstitutablePathInfo sub;
         sub.references = info.references;
         sub.downloadSize = 0;
@@ -246,7 +246,7 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
         infos.emplace(storePath, sub);
     }
 
-    localStore->querySubstitutablePathInfos(left, infos);
+    //(*localStore)->querySubstitutablePathInfos(left, infos);
 }
 
 void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
@@ -258,12 +258,12 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
 
         if (isValidPath(storePath)) continue;
 
-        localStore->addTempRoot(storePath);
+        (*localStore)->addTempRoot(storePath);
 
-        if (!localStore->isValidPath(storePath))
-            localStore->ensurePath(storePath);
+        if (!(*localStore)->isValidPath(storePath))
+            (*localStore)->ensurePath(storePath);
 
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
+        ValidPathInfo info = (*localStore)->queryPathInfo(storePath);
 
         for (auto & ref : info.references)
             if (ref != storePath)
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 8b2d4a81..4d02b3b2 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -5,6 +5,7 @@
 
 #include "lru-cache.hh"
 #include "sync.hh"
+#include "pool.hh"
 
 #include <atomic>
 
@@ -15,7 +16,8 @@ struct NarInfo;
 /* While BinaryCacheStore is thread-safe, LocalStore and RemoteStore
    aren't. Until they are, use a factory to produce a thread-local
    local store. */
-typedef std::function<ref<Store>()> StoreFactory;
+typedef Pool<nix::ref<nix::Store>> StorePool;
+typedef std::function<StorePool::Handle()> StoreFactory;
 
 class BinaryCacheStore : public Store
 {
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 8080af7a..6fd6e2f8 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -18,6 +18,7 @@ using namespace nix;
 
 
 State::State()
+    : localStorePool([]() { return std::make_shared<ref<Store>>(openStore()); })
 {
     hydraData = getEnv("HYDRA_DATA");
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
@@ -26,9 +27,10 @@ State::State()
 }
 
 
-ref<Store> State::getLocalStore()
+StorePool::Handle State::getLocalStore()
 {
-    return openStore(); // FIXME: pool
+    auto conn(localStorePool.get());
+    return conn;
 }
 
 
@@ -733,10 +735,10 @@ void State::run(BuildID buildOne)
 #endif
 
     auto store = std::make_shared<S3BinaryCacheStore>(
-        []() { return openStore(); },
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "nix-test-cache-3");;
+        [this]() { return this->getLocalStore(); },
+        "/home/eelco/hydra/secret",
+        "/home/eelco/hydra/public",
+        "nix-test-cache");
     store->init();
     _destStore = store;
 
diff --git a/src/hydra-queue-runner/pool.hh b/src/hydra-queue-runner/pool.hh
index a1cd3977..83d947e3 100644
--- a/src/hydra-queue-runner/pool.hh
+++ b/src/hydra-queue-runner/pool.hh
@@ -2,6 +2,7 @@
 
 #include <memory>
 #include <list>
+#include <functional>
 
 #include "sync.hh"
 
@@ -25,7 +26,14 @@
 template <class R>
 class Pool
 {
+public:
+
+    typedef std::function<std::shared_ptr<R>()> Factory;
+
 private:
+
+    Factory factory;
+
     struct State
     {
         unsigned int count = 0;
@@ -36,6 +44,10 @@ private:
 
 public:
 
+    Pool(const Factory & factory = []() { return std::make_shared<R>(); })
+        : factory(factory)
+    { }
+
     class Handle
     {
     private:
@@ -74,7 +86,7 @@ public:
         }
         /* Note: we don't hold the lock while creating a new instance,
            because creation might take a long time. */
-        return Handle(*this, std::make_shared<R>());
+        return Handle(*this, factory());
     }
 
     unsigned int count()
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index f7e40827..f3970407 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -36,7 +36,7 @@ void State::queueMonitorLoop()
     unsigned int lastBuildId = 0;
 
     while (true) {
-        bool done = getQueuedBuilds(*conn, localStore, destStore, lastBuildId);
+        bool done = getQueuedBuilds(*conn, *localStore, destStore, lastBuildId);
 
         /* Sleep until we get notification from the database about an
            event. */
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index eb9d6457..a524670d 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -16,6 +16,8 @@
 #include "store-api.hh"
 #include "derivations.hh"
 
+#include "binary-cache-store.hh" // FIXME
+
 
 typedef unsigned int BuildID;
 
@@ -346,6 +348,9 @@ private:
 
     std::atomic<time_t> lastDispatcherCheck{0};
 
+    /* Pool of local stores. */
+    nix::StorePool localStorePool;
+
     /* Destination store. */
     std::shared_ptr<nix::Store> _destStore;
 
@@ -356,7 +361,7 @@ private:
 
     /* Return a store object that can access derivations produced by
        hydra-evaluator. */
-    nix::ref<nix::Store> getLocalStore();
+    nix::StorePool::Handle getLocalStore();
 
     /* Return a store object to store build results. */
     nix::ref<nix::Store> getDestStore();

From 5668aa5f71aa2862534830e98444921470ef97e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Sat, 20 Feb 2016 10:35:16 +0100
Subject: [PATCH 18/40] After uploading a .narinfo, add it to the LRU cache

---
 src/hydra-queue-runner/binary-cache-store.cc | 32 ++++++++++++--------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index f4e9b652..d5c4d679 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -50,33 +50,33 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     auto narInfoFile = narInfoFileFor(info.path);
     if (fileExists(narInfoFile)) return;
 
-    NarInfo narInfo(info);
+    auto narInfo = make_ref<NarInfo>(info);
 
-    narInfo.narSize = nar.size();
-    narInfo.narHash = hashString(htSHA256, nar);
+    narInfo->narSize = nar.size();
+    narInfo->narHash = hashString(htSHA256, nar);
 
-    if (info.narHash.type != htUnknown && info.narHash != narInfo.narHash)
+    if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     /* Compress the NAR. */
-    narInfo.compression = "xz";
+    narInfo->compression = "xz";
     auto now1 = std::chrono::steady_clock::now();
     string narXz = compressXZ(nar);
     auto now2 = std::chrono::steady_clock::now();
-    narInfo.fileHash = hashString(htSHA256, narXz);
-    narInfo.fileSize = narXz.size();
+    narInfo->fileHash = hashString(htSHA256, narXz);
+    narInfo->fileSize = narXz.size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
-        % narInfo.path % narInfo.narSize
+        % narInfo->path % narInfo->narSize
         % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
         % duration);
 
     /* Atomically write the NAR file. */
-    narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz";
-    if (!fileExists(narInfo.url)) {
+    narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";
+    if (!fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo.url, narXz);
+        upsertFile(narInfo->url, narXz);
     } else
         stats.narWriteAverted++;
 
@@ -85,9 +85,15 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo.sign(*secretKey);
+    if (secretKey) narInfo->sign(*secretKey);
 
-    upsertFile(narInfoFile, narInfo.to_string());
+    upsertFile(narInfoFile, narInfo->to_string());
+
+    {
+        auto state_(state.lock());
+        state_->narInfoCache.upsert(narInfo->path, narInfo);
+        stats.narInfoCacheSize = state_->narInfoCache.size();
+    }
 
     stats.narInfoWrite++;
 }

From 94817d77d92f24e247b284bb1fd21269a7bf15d6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 22 Feb 2016 17:21:39 +0100
Subject: [PATCH 19/40] BinaryCacheStore: Respect build-use-substitutes

---
 src/hydra-queue-runner/binary-cache-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index d5c4d679..c76670a5 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -252,7 +252,8 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
         infos.emplace(storePath, sub);
     }
 
-    //(*localStore)->querySubstitutablePathInfos(left, infos);
+    if (settings.useSubstitutes)
+        (*localStore)->querySubstitutablePathInfos(left, infos);
 }
 
 void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)

From 6c3ae3664899e218345cdf1a55af45691af538b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 22 Feb 2016 17:23:06 +0100
Subject: [PATCH 20/40] hydra-queue-runner: Get store mode configuration from
 hydra.conf

To use the local Nix store (default):

  store_mode = direct

To use a local binary cache:

  store_mode = local-binary-cache
  binary_cache_dir = /var/lib/hydra/binary-cache

To use an S3 bucket:

  store_mode = s3-binary-cache
  binary_cache_s3_bucket = my-nix-bucket

Also, respect binary_cache_{secret,public}_key_file for signing the
binary cache.
---
 src/hydra-queue-runner/hydra-queue-runner.cc | 62 ++++++++++++++++----
 src/hydra-queue-runner/state.hh              |  2 +
 2 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 6fd6e2f8..a3ca0221 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -23,6 +23,25 @@ State::State()
     hydraData = getEnv("HYDRA_DATA");
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
 
+    /* Read hydra.conf. */
+    auto hydraConfigFile = getEnv("HYDRA_CONFIG");
+    if (pathExists(hydraConfigFile)) {
+
+        for (auto line : tokenizeString<Strings>(readFile(hydraConfigFile), "\n")) {
+            line = trim(string(line, 0, line.find('#')));
+
+            auto eq = line.find('=');
+            if (eq == std::string::npos) continue;
+
+            auto key = trim(std::string(line, 0, eq));
+            auto value = trim(std::string(line, eq + 1));
+
+            if (key == "") continue;
+
+            hydraConfig[key] = value;
+        }
+    }
+
     logDir = canonPath(hydraData + "/build-logs");
 }
 
@@ -727,20 +746,37 @@ void State::run(BuildID buildOne)
     if (!lock)
         throw Error("hydra-queue-runner is already running");
 
-#if 0
-    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "/tmp/binary-cache");
-#endif
+    auto storeMode = hydraConfig["store_mode"];
 
-    auto store = std::make_shared<S3BinaryCacheStore>(
-        [this]() { return this->getLocalStore(); },
-        "/home/eelco/hydra/secret",
-        "/home/eelco/hydra/public",
-        "nix-test-cache");
-    store->init();
-    _destStore = store;
+    if (storeMode == "direct" || storeMode == "") {
+        _destStore = openStore();
+    }
+
+    else if (storeMode == "local-binary-cache") {
+        auto dir = hydraConfig["binary_cache_dir"];
+        if (dir == "")
+            throw Error("you must set ‘binary_cache_dir’ in hydra.conf");
+        auto store = make_ref<LocalBinaryCacheStore>(
+            [this]() { return this->getLocalStore(); },
+            "/home/eelco/Misc/Keys/test.nixos.org/secret",
+            "/home/eelco/Misc/Keys/test.nixos.org/public",
+            dir);
+        store->init();
+        _destStore = std::shared_ptr<LocalBinaryCacheStore>(store);
+    }
+
+    else if (storeMode == "s3-binary-cache") {
+        auto bucketName = hydraConfig["binary_cache_s3_bucket"];
+        if (bucketName == "")
+            throw Error("you must set ‘binary_cache_s3_bucket’ in hydra.conf");
+        auto store = make_ref<S3BinaryCacheStore>(
+            [this]() { return this->getLocalStore(); },
+            hydraConfig["binary_cache_secret_key_file"],
+            hydraConfig["binary_cache_public_key_file"],
+            bucketName);
+        store->init();
+        _destStore = std::shared_ptr<S3BinaryCacheStore>(store);
+    }
 
     {
         auto conn(dbPool.get());
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index a524670d..58abe6ec 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -262,6 +262,8 @@ private:
 
     nix::Path hydraData, logDir;
 
+    std::map<std::string, std::string> hydraConfig;
+
     /* The queued builds. */
     typedef std::map<BuildID, Build::ptr> Builds;
     Sync<Builds> builds;

From 7b509237cded1e5b26885613ce96d4ffb2870b21 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 22 Feb 2016 18:05:15 +0100
Subject: [PATCH 21/40] Bleh Automake

---
 src/hydra-queue-runner/Makefile.am | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 6d526c89..83242759 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -5,7 +5,8 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c
  build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
  binary-cache-store.hh binary-cache-store.cc \
  local-binary-cache-store.hh local-binary-cache-store.cc \
- s3-binary-cache-store.hh s3-binary-cache-store.cc
+ s3-binary-cache-store.hh s3-binary-cache-store.cc \
+ lru-cache.hh
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3

From 8321a3eb2744501bc08238ef500acddc1ad8956d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Wed, 24 Feb 2016 14:04:31 +0100
Subject: [PATCH 22/40] Sync with Nix

---
 src/hydra-queue-runner/binary-cache-store.cc  | 25 ++---
 src/hydra-queue-runner/binary-cache-store.hh  | 10 +-
 src/hydra-queue-runner/hydra-queue-runner.cc  | 18 ++--
 .../local-binary-cache-store.cc               |  4 +-
 .../local-binary-cache-store.hh               |  2 +-
 src/hydra-queue-runner/pool.hh                | 97 -------------------
 src/hydra-queue-runner/queue-monitor.cc       |  2 +-
 .../s3-binary-cache-store.cc                  |  4 +-
 .../s3-binary-cache-store.hh                  |  2 +-
 src/hydra-queue-runner/state.hh               | 39 ++++----
 src/hydra-queue-runner/sync.hh                | 74 --------------
 src/hydra-queue-runner/token-server.hh        |  2 +-
 12 files changed, 50 insertions(+), 229 deletions(-)
 delete mode 100644 src/hydra-queue-runner/pool.hh
 delete mode 100644 src/hydra-queue-runner/sync.hh

diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
index c76670a5..6bea0e6c 100644
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ b/src/hydra-queue-runner/binary-cache-store.cc
@@ -12,9 +12,9 @@
 
 namespace nix {
 
-BinaryCacheStore::BinaryCacheStore(const StoreFactory & storeFactory,
+BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,
     const Path & secretKeyFile, const Path & publicKeyFile)
-    : storeFactory(storeFactory)
+    : localStore(localStore)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@@ -237,14 +237,14 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     PathSet left;
 
-    auto localStore = storeFactory();
+    if (!localStore) return;
 
     for (auto & storePath : paths) {
-        if (!(*localStore)->isValidPath(storePath)) {
+        if (!localStore->isValidPath(storePath)) {
             left.insert(storePath);
             continue;
         }
-        ValidPathInfo info = (*localStore)->queryPathInfo(storePath);
+        ValidPathInfo info = localStore->queryPathInfo(storePath);
         SubstitutablePathInfo sub;
         sub.references = info.references;
         sub.downloadSize = 0;
@@ -253,24 +253,25 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
     }
 
     if (settings.useSubstitutes)
-        (*localStore)->querySubstitutablePathInfos(left, infos);
+        localStore->querySubstitutablePathInfos(left, infos);
 }
 
 void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
-    auto localStore = storeFactory();
-
     for (auto & storePath : paths) {
         assert(!isDerivation(storePath));
 
         if (isValidPath(storePath)) continue;
 
-        (*localStore)->addTempRoot(storePath);
+        if (!localStore)
+            throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath);
 
-        if (!(*localStore)->isValidPath(storePath))
-            (*localStore)->ensurePath(storePath);
+        localStore->addTempRoot(storePath);
 
-        ValidPathInfo info = (*localStore)->queryPathInfo(storePath);
+        if (!localStore->isValidPath(storePath))
+            localStore->ensurePath(storePath);
+
+        ValidPathInfo info = localStore->queryPathInfo(storePath);
 
         for (auto & ref : info.references)
             if (ref != storePath)
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
index 4d02b3b2..d02f46de 100644
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ b/src/hydra-queue-runner/binary-cache-store.hh
@@ -13,12 +13,6 @@ namespace nix {
 
 struct NarInfo;
 
-/* While BinaryCacheStore is thread-safe, LocalStore and RemoteStore
-   aren't. Until they are, use a factory to produce a thread-local
-   local store. */
-typedef Pool<nix::ref<nix::Store>> StorePool;
-typedef std::function<StorePool::Handle()> StoreFactory;
-
 class BinaryCacheStore : public Store
 {
 private:
@@ -26,7 +20,7 @@ private:
     std::unique_ptr<SecretKey> secretKey;
     std::unique_ptr<PublicKeys> publicKeys;
 
-    StoreFactory storeFactory;
+    std::shared_ptr<Store> localStore;
 
     struct State
     {
@@ -37,7 +31,7 @@ private:
 
 protected:
 
-    BinaryCacheStore(const StoreFactory & storeFactory,
+    BinaryCacheStore(std::shared_ptr<Store> localStore,
         const Path & secretKeyFile, const Path & publicKeyFile);
 
     virtual bool fileExists(const std::string & path) = 0;
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index a3ca0221..2e4229c2 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -18,7 +18,6 @@ using namespace nix;
 
 
 State::State()
-    : localStorePool([]() { return std::make_shared<ref<Store>>(openStore()); })
 {
     hydraData = getEnv("HYDRA_DATA");
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
@@ -46,10 +45,9 @@ State::State()
 }
 
 
-StorePool::Handle State::getLocalStore()
+ref<Store> State::getLocalStore()
 {
-    auto conn(localStorePool.get());
-    return conn;
+    return ref<Store>(_localStore);
 }
 
 
@@ -748,8 +746,10 @@ void State::run(BuildID buildOne)
 
     auto storeMode = hydraConfig["store_mode"];
 
+    _localStore = openStore();
+
     if (storeMode == "direct" || storeMode == "") {
-        _destStore = openStore();
+        _destStore = _localStore;
     }
 
     else if (storeMode == "local-binary-cache") {
@@ -757,9 +757,9 @@ void State::run(BuildID buildOne)
         if (dir == "")
             throw Error("you must set ‘binary_cache_dir’ in hydra.conf");
         auto store = make_ref<LocalBinaryCacheStore>(
-            [this]() { return this->getLocalStore(); },
-            "/home/eelco/Misc/Keys/test.nixos.org/secret",
-            "/home/eelco/Misc/Keys/test.nixos.org/public",
+            _localStore,
+            hydraConfig["binary_cache_secret_key_file"],
+            hydraConfig["binary_cache_public_key_file"],
             dir);
         store->init();
         _destStore = std::shared_ptr<LocalBinaryCacheStore>(store);
@@ -770,7 +770,7 @@ void State::run(BuildID buildOne)
         if (bucketName == "")
             throw Error("you must set ‘binary_cache_s3_bucket’ in hydra.conf");
         auto store = make_ref<S3BinaryCacheStore>(
-            [this]() { return this->getLocalStore(); },
+            _localStore,
             hydraConfig["binary_cache_secret_key_file"],
             hydraConfig["binary_cache_public_key_file"],
             bucketName);
diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc
index a82e6597..5714688e 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.cc
+++ b/src/hydra-queue-runner/local-binary-cache-store.cc
@@ -2,10 +2,10 @@
 
 namespace nix {
 
-LocalBinaryCacheStore::LocalBinaryCacheStore(const StoreFactory & storeFactory,
+LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const Path & binaryCacheDir)
-    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
     , binaryCacheDir(binaryCacheDir)
 {
 }
diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh
index e29d0ca2..0303ebe7 100644
--- a/src/hydra-queue-runner/local-binary-cache-store.hh
+++ b/src/hydra-queue-runner/local-binary-cache-store.hh
@@ -12,7 +12,7 @@ private:
 
 public:
 
-    LocalBinaryCacheStore(const StoreFactory & storeFactory,
+    LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const Path & binaryCacheDir);
 
diff --git a/src/hydra-queue-runner/pool.hh b/src/hydra-queue-runner/pool.hh
deleted file mode 100644
index 83d947e3..00000000
--- a/src/hydra-queue-runner/pool.hh
+++ /dev/null
@@ -1,97 +0,0 @@
-#pragma once
-
-#include <memory>
-#include <list>
-#include <functional>
-
-#include "sync.hh"
-
-/* This template class implements a simple pool manager of resources
-   of some type R, such as database connections. It is used as
-   follows:
-
-     class Connection { ... };
-
-     Pool<Connection> pool;
-
-     {
-       auto conn(pool.get());
-       conn->exec("select ...");
-     }
-
-   Here, the Connection object referenced by ‘conn’ is automatically
-   returned to the pool when ‘conn’ goes out of scope.
-*/
-
-template <class R>
-class Pool
-{
-public:
-
-    typedef std::function<std::shared_ptr<R>()> Factory;
-
-private:
-
-    Factory factory;
-
-    struct State
-    {
-        unsigned int count = 0;
-        std::list<std::shared_ptr<R>> idle;
-    };
-
-    Sync<State> state;
-
-public:
-
-    Pool(const Factory & factory = []() { return std::make_shared<R>(); })
-        : factory(factory)
-    { }
-
-    class Handle
-    {
-    private:
-        Pool & pool;
-        std::shared_ptr<R> r;
-
-        friend Pool;
-
-        Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { }
-
-    public:
-        Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); }
-
-        Handle(const Handle & l) = delete;
-
-        ~Handle()
-        {
-            auto state_(pool.state.lock());
-            if (r) state_->idle.push_back(r);
-        }
-
-        R * operator -> () { return r.get(); }
-        R & operator * () { return *r; }
-    };
-
-    Handle get()
-    {
-        {
-            auto state_(state.lock());
-            if (!state_->idle.empty()) {
-                auto p = state_->idle.back();
-                state_->idle.pop_back();
-                return Handle(*this, p);
-            }
-            state_->count++;
-        }
-        /* Note: we don't hold the lock while creating a new instance,
-           because creation might take a long time. */
-        return Handle(*this, factory());
-    }
-
-    unsigned int count()
-    {
-        auto state_(state.lock());
-        return state_->count;
-    }
-};
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index f3970407..f7e40827 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -36,7 +36,7 @@ void State::queueMonitorLoop()
     unsigned int lastBuildId = 0;
 
     while (true) {
-        bool done = getQueuedBuilds(*conn, *localStore, destStore, lastBuildId);
+        bool done = getQueuedBuilds(*conn, localStore, destStore, lastBuildId);
 
         /* Sleep until we get notification from the database about an
            event. */
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 3f6b6bf9..4a78033a 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -31,10 +31,10 @@ R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
     return outcome.GetResultWithOwnership();
 }
 
-S3BinaryCacheStore::S3BinaryCacheStore(const StoreFactory & storeFactory,
+S3BinaryCacheStore::S3BinaryCacheStore(std::shared_ptr<Store> localStore,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const std::string & bucketName)
-    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
     , bucketName(bucketName)
     , config(makeConfig())
     , client(make_ref<Aws::S3::S3Client>(*config))
diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh
index a0dd7813..1ba78dce 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.hh
+++ b/src/hydra-queue-runner/s3-binary-cache-store.hh
@@ -20,7 +20,7 @@ private:
 
 public:
 
-    S3BinaryCacheStore(const StoreFactory & storeFactory,
+    S3BinaryCacheStore(std::shared_ptr<Store> localStore,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const std::string & bucketName);
 
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 58abe6ec..01dfd650 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -80,7 +80,7 @@ private:
     std::atomic<unsigned int> shares{1};
 
     /* The start time and duration of the most recent build steps. */
-    Sync<std::map<time_t, time_t>> steps;
+    nix::Sync<std::map<time_t, time_t>> steps;
 
 public:
 
@@ -187,7 +187,7 @@ struct Step
 
     std::atomic_bool finished{false}; // debugging
 
-    Sync<State> state;
+    nix::Sync<State> state;
 
     ~Step()
     {
@@ -227,7 +227,7 @@ struct Machine
             system_time lastFailure, disabledUntil;
             unsigned int consecutiveFailures;
         };
-        Sync<ConnectInfo> connectInfo;
+        nix::Sync<ConnectInfo> connectInfo;
 
         /* Mutex to prevent multiple threads from sending data to the
            same machine (which would be inefficient). */
@@ -266,33 +266,33 @@ private:
 
     /* The queued builds. */
     typedef std::map<BuildID, Build::ptr> Builds;
-    Sync<Builds> builds;
+    nix::Sync<Builds> builds;
 
     /* The jobsets. */
     typedef std::map<std::pair<std::string, std::string>, Jobset::ptr> Jobsets;
-    Sync<Jobsets> jobsets;
+    nix::Sync<Jobsets> jobsets;
 
     /* All active or pending build steps (i.e. dependencies of the
        queued builds). Note that these are weak pointers. Steps are
        kept alive by being reachable from Builds or by being in
        progress. */
     typedef std::map<nix::Path, Step::wptr> Steps;
-    Sync<Steps> steps;
+    nix::Sync<Steps> steps;
 
     /* Build steps that have no unbuilt dependencies. */
     typedef std::list<Step::wptr> Runnable;
-    Sync<Runnable> runnable;
+    nix::Sync<Runnable> runnable;
 
     /* CV for waking up the dispatcher. */
-    Sync<bool> dispatcherWakeup;
-    std::condition_variable_any dispatcherWakeupCV;
+    nix::Sync<bool> dispatcherWakeup;
+    std::condition_variable dispatcherWakeupCV;
 
     /* PostgreSQL connection pool. */
-    Pool<Connection> dbPool;
+    nix::Pool<Connection> dbPool;
 
     /* The build machines. */
     typedef std::map<std::string, Machine::ptr> Machines;
-    Sync<Machines> machines; // FIXME: use atomic_shared_ptr
+    nix::Sync<Machines> machines; // FIXME: use atomic_shared_ptr
 
     /* Various stats. */
     time_t startedAt;
@@ -314,16 +314,16 @@ private:
     counter bytesReceived{0};
 
     /* Log compressor work queue. */
-    Sync<std::queue<nix::Path>> logCompressorQueue;
-    std::condition_variable_any logCompressorWakeup;
+    nix::Sync<std::queue<nix::Path>> logCompressorQueue;
+    std::condition_variable logCompressorWakeup;
 
     /* Notification sender work queue. FIXME: if hydra-queue-runner is
        killed before it has finished sending notifications about a
        build, then the notifications may be lost. It would be better
        to mark builds with pending notification in the database. */
     typedef std::pair<BuildID, std::vector<BuildID>> NotificationItem;
-    Sync<std::queue<NotificationItem>> notificationSenderQueue;
-    std::condition_variable_any notificationSenderWakeup;
+    nix::Sync<std::queue<NotificationItem>> notificationSenderQueue;
+    std::condition_variable notificationSenderWakeup;
 
     /* Specific build to do for --build-one (testing only). */
     BuildID buildOne;
@@ -336,7 +336,7 @@ private:
         std::chrono::seconds waitTime; // time runnable steps have been waiting
     };
 
-    Sync<std::map<std::string, MachineType>> machineTypes;
+    nix::Sync<std::map<std::string, MachineType>> machineTypes;
 
     struct MachineReservation
     {
@@ -350,10 +350,7 @@ private:
 
     std::atomic<time_t> lastDispatcherCheck{0};
 
-    /* Pool of local stores. */
-    nix::StorePool localStorePool;
-
-    /* Destination store. */
+    std::shared_ptr<nix::Store> _localStore;
     std::shared_ptr<nix::Store> _destStore;
 
 public:
@@ -363,7 +360,7 @@ private:
 
     /* Return a store object that can access derivations produced by
        hydra-evaluator. */
-    nix::StorePool::Handle getLocalStore();
+    nix::ref<nix::Store> getLocalStore();
 
     /* Return a store object to store build results. */
     nix::ref<nix::Store> getDestStore();
diff --git a/src/hydra-queue-runner/sync.hh b/src/hydra-queue-runner/sync.hh
deleted file mode 100644
index 4d53a6ee..00000000
--- a/src/hydra-queue-runner/sync.hh
+++ /dev/null
@@ -1,74 +0,0 @@
-#pragma once
-
-#include <mutex>
-#include <condition_variable>
-#include <cassert>
-
-/* This template class ensures synchronized access to a value of type
-   T. It is used as follows:
-
-     struct Data { int x; ... };
-
-     Sync<Data> data;
-
-     {
-       auto data_(data.lock());
-       data_->x = 123;
-     }
-
-   Here, "data" is automatically unlocked when "data_" goes out of
-   scope.
-*/
-
-template<class T>
-class Sync
-{
-private:
-    std::mutex mutex;
-    T data;
-
-public:
-
-    Sync() { }
-    Sync(const T & data) : data(data) { }
-
-    class Lock
-    {
-    private:
-        Sync * s;
-        friend Sync;
-        Lock(Sync * s) : s(s) { s->mutex.lock(); }
-    public:
-        Lock(Lock && l) : s(l.s) { l.s = 0; }
-        Lock(const Lock & l) = delete;
-        ~Lock() { if (s) s->mutex.unlock(); }
-        T * operator -> () { return &s->data; }
-        T & operator * () { return s->data; }
-
-        /* FIXME: performance impact of condition_variable_any? */
-        void wait(std::condition_variable_any & cv)
-        {
-            assert(s);
-            cv.wait(s->mutex);
-        }
-
-        template<class Rep, class Period, class Predicate>
-        bool wait_for(std::condition_variable_any & cv,
-            const std::chrono::duration<Rep, Period> & duration,
-            Predicate pred)
-        {
-            assert(s);
-            return cv.wait_for(s->mutex, duration, pred);
-        }
-
-        template<class Clock, class Duration>
-        std::cv_status wait_until(std::condition_variable_any & cv,
-            const std::chrono::time_point<Clock, Duration> & duration)
-        {
-            assert(s);
-            return cv.wait_until(s->mutex, duration);
-        }
-    };
-
-    Lock lock() { return Lock(this); }
-};
diff --git a/src/hydra-queue-runner/token-server.hh b/src/hydra-queue-runner/token-server.hh
index 2ff748e3..d4f5f843 100644
--- a/src/hydra-queue-runner/token-server.hh
+++ b/src/hydra-queue-runner/token-server.hh
@@ -14,7 +14,7 @@ class TokenServer
     unsigned int maxTokens;
 
     Sync<unsigned int> curTokens{0};
-    std::condition_variable_any wakeup;
+    std::condition_variable wakeup;
 
 public:
     TokenServer(unsigned int maxTokens) : maxTokens(maxTokens) { }

From 8e24ad6f0d5ccc0741c6b3e65409531799859748 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 25 Feb 2016 10:58:31 +0100
Subject: [PATCH 23/40] Sync with Nix

---
 src/hydra-queue-runner/Makefile.am            |   7 +-
 src/hydra-queue-runner/binary-cache-store.cc  | 292 ------------------
 src/hydra-queue-runner/binary-cache-store.hh  | 170 ----------
 .../local-binary-cache-store.cc               |  44 ---
 .../local-binary-cache-store.hh               |  31 --
 src/hydra-queue-runner/lru-cache.hh           |  80 -----
 6 files changed, 2 insertions(+), 622 deletions(-)
 delete mode 100644 src/hydra-queue-runner/binary-cache-store.cc
 delete mode 100644 src/hydra-queue-runner/binary-cache-store.hh
 delete mode 100644 src/hydra-queue-runner/local-binary-cache-store.cc
 delete mode 100644 src/hydra-queue-runner/local-binary-cache-store.hh
 delete mode 100644 src/hydra-queue-runner/lru-cache.hh

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 83242759..2cf4fcdb 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -2,11 +2,8 @@ bin_PROGRAMS = hydra-queue-runner
 
 hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
  builder.cc build-result.cc build-remote.cc  \
- build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \
- binary-cache-store.hh binary-cache-store.cc \
- local-binary-cache-store.hh local-binary-cache-store.cc \
- s3-binary-cache-store.hh s3-binary-cache-store.cc \
- lru-cache.hh
+ build-result.hh counter.hh token-server.hh state.hh db.hh \
+ s3-binary-cache-store.hh s3-binary-cache-store.cc
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3
diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc
deleted file mode 100644
index 6bea0e6c..00000000
--- a/src/hydra-queue-runner/binary-cache-store.cc
+++ /dev/null
@@ -1,292 +0,0 @@
-#include "binary-cache-store.hh"
-#include "sync.hh"
-
-#include "archive.hh"
-#include "compression.hh"
-#include "derivations.hh"
-#include "globals.hh"
-#include "nar-info.hh"
-#include "worker-protocol.hh"
-
-#include <chrono>
-
-namespace nix {
-
-BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile, const Path & publicKeyFile)
-    : localStore(localStore)
-{
-    if (secretKeyFile != "")
-        secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
-
-    if (publicKeyFile != "") {
-        publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);
-        auto key = PublicKey(readFile(publicKeyFile));
-        publicKeys->emplace(key.name, key);
-    }
-}
-
-void BinaryCacheStore::init()
-{
-    std::string cacheInfoFile = "nix-cache-info";
-    if (!fileExists(cacheInfoFile))
-        upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
-}
-
-const BinaryCacheStore::Stats & BinaryCacheStore::getStats()
-{
-    return stats;
-}
-
-Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
-{
-    assertStorePath(storePath);
-    return storePathToHash(storePath) + ".narinfo";
-}
-
-void BinaryCacheStore::addToCache(const ValidPathInfo & info,
-    const string & nar)
-{
-    auto narInfoFile = narInfoFileFor(info.path);
-    if (fileExists(narInfoFile)) return;
-
-    auto narInfo = make_ref<NarInfo>(info);
-
-    narInfo->narSize = nar.size();
-    narInfo->narHash = hashString(htSHA256, nar);
-
-    if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
-        throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
-
-    /* Compress the NAR. */
-    narInfo->compression = "xz";
-    auto now1 = std::chrono::steady_clock::now();
-    string narXz = compressXZ(nar);
-    auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, narXz);
-    narInfo->fileSize = narXz.size();
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
-        % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
-        % duration);
-
-    /* Atomically write the NAR file. */
-    narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";
-    if (!fileExists(narInfo->url)) {
-        stats.narWrite++;
-        upsertFile(narInfo->url, narXz);
-    } else
-        stats.narWriteAverted++;
-
-    stats.narWriteBytes += nar.size();
-    stats.narWriteCompressedBytes += narXz.size();
-    stats.narWriteCompressionTimeMs += duration;
-
-    /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*secretKey);
-
-    upsertFile(narInfoFile, narInfo->to_string());
-
-    {
-        auto state_(state.lock());
-        state_->narInfoCache.upsert(narInfo->path, narInfo);
-        stats.narInfoCacheSize = state_->narInfoCache.size();
-    }
-
-    stats.narInfoWrite++;
-}
-
-NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
-{
-    {
-        auto state_(state.lock());
-        auto res = state_->narInfoCache.get(storePath);
-        if (res) {
-            stats.narInfoReadAverted++;
-            return **res;
-        }
-    }
-
-    auto narInfoFile = narInfoFileFor(storePath);
-    auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
-    assert(narInfo->path == storePath);
-
-    stats.narInfoRead++;
-
-    if (publicKeys) {
-        if (!narInfo->checkSignature(*publicKeys))
-            throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
-    }
-
-    {
-        auto state_(state.lock());
-        state_->narInfoCache.upsert(storePath, narInfo);
-        stats.narInfoCacheSize = state_->narInfoCache.size();
-    }
-
-    return *narInfo;
-}
-
-bool BinaryCacheStore::isValidPath(const Path & storePath)
-{
-    return fileExists(narInfoFileFor(storePath));
-}
-
-void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)
-{
-    assert(!sign);
-
-    auto res = readNarInfo(storePath);
-
-    auto nar = getFile(res.url);
-
-    stats.narRead++;
-    stats.narReadCompressedBytes += nar.size();
-
-    /* Decompress the NAR. FIXME: would be nice to have the remote
-       side do this. */
-    if (res.compression == "none")
-        ;
-    else if (res.compression == "xz")
-        nar = decompressXZ(nar);
-    else
-        throw Error(format("unknown NAR compression type ‘%1%’") % nar);
-
-    stats.narReadBytes += nar.size();
-
-    printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
-
-    assert(nar.size() % 8 == 0);
-
-    sink((unsigned char *) nar.c_str(), nar.size());
-
-    // FIXME: check integrity of NAR.
-
-    sink << exportMagic << storePath << res.references << res.deriver << 0;
-}
-
-Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
-{
-    assert(!requireSignature);
-    Paths res;
-    while (true) {
-        unsigned long long n = readLongLong(source);
-        if (n == 0) break;
-        if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
-        res.push_back(importPath(source));
-    }
-    return res;
-}
-
-struct TeeSource : Source
-{
-    Source & readSource;
-    std::string data;
-    TeeSource(Source & readSource) : readSource(readSource)
-    {
-    }
-    size_t read(unsigned char * data, size_t len)
-    {
-        size_t n = readSource.read(data, len);
-        this->data.append((char *) data, n);
-        return n;
-    }
-};
-
-struct NopSink : ParseSink
-{
-};
-
-Path BinaryCacheStore::importPath(Source & source)
-{
-    /* FIXME: some cut&paste of LocalStore::importPath(). */
-
-    /* Extract the NAR from the source. */
-    TeeSource tee(source);
-    NopSink sink;
-    parseDump(sink, tee);
-
-    uint32_t magic = readInt(source);
-    if (magic != exportMagic)
-        throw Error("Nix archive cannot be imported; wrong format");
-
-    ValidPathInfo info;
-    info.path = readStorePath(source);
-
-    info.references = readStorePaths<PathSet>(source);
-
-    readString(source); // deriver, don't care
-
-    bool haveSignature = readInt(source) == 1;
-    assert(!haveSignature);
-
-    addToCache(info, tee.data);
-
-    return info.path;
-}
-
-ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
-{
-    return ValidPathInfo(readNarInfo(storePath));
-}
-
-void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
-    SubstitutablePathInfos & infos)
-{
-    PathSet left;
-
-    if (!localStore) return;
-
-    for (auto & storePath : paths) {
-        if (!localStore->isValidPath(storePath)) {
-            left.insert(storePath);
-            continue;
-        }
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
-        SubstitutablePathInfo sub;
-        sub.references = info.references;
-        sub.downloadSize = 0;
-        sub.narSize = info.narSize;
-        infos.emplace(storePath, sub);
-    }
-
-    if (settings.useSubstitutes)
-        localStore->querySubstitutablePathInfos(left, infos);
-}
-
-void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
-{
-    for (auto & storePath : paths) {
-        assert(!isDerivation(storePath));
-
-        if (isValidPath(storePath)) continue;
-
-        if (!localStore)
-            throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath);
-
-        localStore->addTempRoot(storePath);
-
-        if (!localStore->isValidPath(storePath))
-            localStore->ensurePath(storePath);
-
-        ValidPathInfo info = localStore->queryPathInfo(storePath);
-
-        for (auto & ref : info.references)
-            if (ref != storePath)
-                ensurePath(ref);
-
-        StringSink sink;
-        dumpPath(storePath, sink);
-
-        addToCache(info, sink.s);
-    }
-}
-
-void BinaryCacheStore::ensurePath(const Path & path)
-{
-    buildPaths({path});
-}
-
-}
diff --git a/src/hydra-queue-runner/binary-cache-store.hh b/src/hydra-queue-runner/binary-cache-store.hh
deleted file mode 100644
index d02f46de..00000000
--- a/src/hydra-queue-runner/binary-cache-store.hh
+++ /dev/null
@@ -1,170 +0,0 @@
-#pragma once
-
-#include "crypto.hh"
-#include "store-api.hh"
-
-#include "lru-cache.hh"
-#include "sync.hh"
-#include "pool.hh"
-
-#include <atomic>
-
-namespace nix {
-
-struct NarInfo;
-
-class BinaryCacheStore : public Store
-{
-private:
-
-    std::unique_ptr<SecretKey> secretKey;
-    std::unique_ptr<PublicKeys> publicKeys;
-
-    std::shared_ptr<Store> localStore;
-
-    struct State
-    {
-        LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024};
-    };
-
-    Sync<State> state;
-
-protected:
-
-    BinaryCacheStore(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile, const Path & publicKeyFile);
-
-    virtual bool fileExists(const std::string & path) = 0;
-
-    virtual void upsertFile(const std::string & path, const std::string & data) = 0;
-
-    virtual std::string getFile(const std::string & path) = 0;
-
-public:
-
-    virtual void init();
-
-    struct Stats
-    {
-        std::atomic<uint64_t> narInfoRead{0};
-        std::atomic<uint64_t> narInfoReadAverted{0};
-        std::atomic<uint64_t> narInfoWrite{0};
-        std::atomic<uint64_t> narInfoCacheSize{0};
-        std::atomic<uint64_t> narRead{0};
-        std::atomic<uint64_t> narReadBytes{0};
-        std::atomic<uint64_t> narReadCompressedBytes{0};
-        std::atomic<uint64_t> narWrite{0};
-        std::atomic<uint64_t> narWriteAverted{0};
-        std::atomic<uint64_t> narWriteBytes{0};
-        std::atomic<uint64_t> narWriteCompressedBytes{0};
-        std::atomic<uint64_t> narWriteCompressionTimeMs{0};
-    };
-
-    const Stats & getStats();
-
-private:
-
-    Stats stats;
-
-    std::string narInfoFileFor(const Path & storePath);
-
-    void addToCache(const ValidPathInfo & info, const string & nar);
-
-protected:
-
-    NarInfo readNarInfo(const Path & storePath);
-
-public:
-
-    bool isValidPath(const Path & path) override;
-
-    PathSet queryValidPaths(const PathSet & paths) override
-    { abort(); }
-
-    PathSet queryAllValidPaths() override
-    { abort(); }
-
-    ValidPathInfo queryPathInfo(const Path & path) override;
-
-    Hash queryPathHash(const Path & path) override
-    { abort(); }
-
-    void queryReferrers(const Path & path,
-        PathSet & referrers) override
-    { abort(); }
-
-    Path queryDeriver(const Path & path) override
-    { abort(); }
-
-    PathSet queryValidDerivers(const Path & path) override
-    { abort(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { abort(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { abort(); }
-
-    Path queryPathFromHashPart(const string & hashPart) override
-    { abort(); }
-
-    PathSet querySubstitutablePaths(const PathSet & paths) override
-    { abort(); }
-
-    void querySubstitutablePathInfos(const PathSet & paths,
-        SubstitutablePathInfos & infos) override;
-
-    Path addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, bool repair = false) override
-    { abort(); }
-
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair = false) override
-    { abort(); }
-
-    void exportPath(const Path & path, bool sign,
-        Sink & sink) override;
-
-    Paths importPaths(bool requireSignature, Source & source) override;
-
-    Path importPath(Source & source);
-
-    void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
-
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode = bmNormal) override
-    { abort(); }
-
-    void ensurePath(const Path & path) override;
-
-    void addTempRoot(const Path & path) override
-    { abort(); }
-
-    void addIndirectRoot(const Path & path) override
-    { abort(); }
-
-    void syncWithGC() override
-    { }
-
-    Roots findRoots() override
-    { abort(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { abort(); }
-
-    PathSet queryFailedPaths() override
-    { return PathSet(); }
-
-    void clearFailedPaths(const PathSet & paths) override
-    { }
-
-    void optimiseStore() override
-    { }
-
-    bool verifyStore(bool checkContents, bool repair) override
-    { return true; }
-
-};
-
-}
diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc
deleted file mode 100644
index 5714688e..00000000
--- a/src/hydra-queue-runner/local-binary-cache-store.cc
+++ /dev/null
@@ -1,44 +0,0 @@
-#include "local-binary-cache-store.hh"
-
-namespace nix {
-
-LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile, const Path & publicKeyFile,
-    const Path & binaryCacheDir)
-    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
-    , binaryCacheDir(binaryCacheDir)
-{
-}
-
-void LocalBinaryCacheStore::init()
-{
-    createDirs(binaryCacheDir + "/nar");
-    BinaryCacheStore::init();
-}
-
-static void atomicWrite(const Path & path, const std::string & s)
-{
-    Path tmp = path + ".tmp." + std::to_string(getpid());
-    AutoDelete del(tmp, false);
-    writeFile(tmp, s);
-    if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
-    del.cancel();
-}
-
-bool LocalBinaryCacheStore::fileExists(const std::string & path)
-{
-    return pathExists(binaryCacheDir + "/" + path);
-}
-
-void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
-{
-    atomicWrite(binaryCacheDir + "/" + path, data);
-}
-
-std::string LocalBinaryCacheStore::getFile(const std::string & path)
-{
-    return readFile(binaryCacheDir + "/" + path);
-}
-
-}
diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh
deleted file mode 100644
index 0303ebe7..00000000
--- a/src/hydra-queue-runner/local-binary-cache-store.hh
+++ /dev/null
@@ -1,31 +0,0 @@
-#pragma once
-
-#include "binary-cache-store.hh"
-
-namespace nix {
-
-class LocalBinaryCacheStore : public BinaryCacheStore
-{
-private:
-
-    Path binaryCacheDir;
-
-public:
-
-    LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile, const Path & publicKeyFile,
-        const Path & binaryCacheDir);
-
-    void init() override;
-
-protected:
-
-    bool fileExists(const std::string & path) override;
-
-    void upsertFile(const std::string & path, const std::string & data) override;
-
-    std::string getFile(const std::string & path) override;
-
-};
-
-}
diff --git a/src/hydra-queue-runner/lru-cache.hh b/src/hydra-queue-runner/lru-cache.hh
deleted file mode 100644
index 113411bd..00000000
--- a/src/hydra-queue-runner/lru-cache.hh
+++ /dev/null
@@ -1,80 +0,0 @@
-#pragma once
-
-#include <map>
-#include <list>
-
-/* A simple least-recently used cache. Not thread-safe. */
-template<typename Key, typename Value>
-class LRUCache
-{
-private:
-
-    size_t maxSize;
-
-    // Stupid wrapper to get around circular dependency between Data
-    // and LRU.
-    struct LRUIterator;
-
-    using Data = std::map<Key, std::pair<LRUIterator, Value>>;
-    using LRU = std::list<typename Data::iterator>;
-
-    struct LRUIterator { typename LRU::iterator it; };
-
-    Data data;
-    LRU lru;
-
-public:
-
-    LRUCache(size_t maxSize) : maxSize(maxSize) { }
-
-    /* Insert or upsert an item in the cache. */
-    void upsert(const Key & key, const Value & value)
-    {
-        erase(key);
-
-        if (data.size() >= maxSize) {
-            /* Retire the oldest item. */
-            auto oldest = lru.begin();
-            data.erase(*oldest);
-            lru.erase(oldest);
-        }
-
-        auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
-        assert(res.second);
-        auto & i(res.first);
-
-        auto j = lru.insert(lru.end(), i);
-
-        i->second.first.it = j;
-    }
-
-    bool erase(const Key & key)
-    {
-        auto i = data.find(key);
-        if (i == data.end()) return false;
-        lru.erase(i->second.first.it);
-        data.erase(i);
-        return true;
-    }
-
-    /* Look up an item in the cache. If it's exists, it becomes the
-       most recently used item. */
-    // FIXME: use boost::optional?
-    Value * get(const Key & key)
-    {
-        auto i = data.find(key);
-        if (i == data.end()) return 0;
-
-        /* Move this item to the back of the LRU list. */
-        lru.erase(i->second.first.it);
-        auto j = lru.insert(lru.end(), i);
-        i->second.first.it = j;
-
-        return &i->second.second;
-    }
-
-    size_t size()
-    {
-        return data.size();
-    }
-};

From 02190b0fefe7b5739c0eac403264c07d15ae3635 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 14:45:03 +0100
Subject: [PATCH 24/40] Support hydra-build-products on binary cache stores

---
 src/hydra-queue-runner/build-result.cc  | 77 ++++++++++---------------
 src/hydra-queue-runner/build-result.hh  |  4 +-
 src/hydra-queue-runner/builder.cc       |  2 +-
 src/hydra-queue-runner/queue-monitor.cc |  2 +-
 4 files changed, 34 insertions(+), 51 deletions(-)

diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc
index f09d13a1..6b2741ea 100644
--- a/src/hydra-queue-runner/build-result.cc
+++ b/src/hydra-queue-runner/build-result.cc
@@ -2,26 +2,13 @@
 #include "store-api.hh"
 #include "util.hh"
 #include "regex.hh"
+#include "fs-accessor.hh"
 
 using namespace nix;
 
 
-static std::tuple<bool, string> secureRead(Path fileName)
-{
-    auto fail = std::make_tuple(false, "");
-
-    if (!pathExists(fileName)) return fail;
-
-    try {
-        /* For security, resolve symlinks. */
-        fileName = canonPath(fileName, true);
-        if (!isInStore(fileName)) return fail;
-        return std::make_tuple(true, readFile(fileName));
-    } catch (Error & e) { return fail; }
-}
-
-
-BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
+BuildOutput getBuildOutput(nix::ref<Store> store,
+    nix::ref<nix::FSAccessor> accessor, const Derivation & drv)
 {
     BuildOutput res;
 
@@ -41,7 +28,6 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
     /* Get build products. */
     bool explicitProducts = false;
 
-#if 0
     Regex regex(
         "(([a-zA-Z0-9_-]+)" // type (e.g. "doc")
         "[[:space:]]+"
@@ -53,14 +39,16 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
 
     for (auto & output : outputs) {
         Path failedFile = output + "/nix-support/failed";
-        if (pathExists(failedFile)) res.failed = true;
+        if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)
+            res.failed = true;
 
-        auto file = secureRead(output + "/nix-support/hydra-build-products");
-        if (!std::get<0>(file)) continue;
+        Path productsFile = output + "/nix-support/hydra-build-products";
+        if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)
+            continue;
 
         explicitProducts = true;
 
-        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
+        for (auto & line : tokenizeString<Strings>(accessor->readFile(productsFile), "\n")) {
             BuildProduct product;
 
             Regex::Subs subs;
@@ -73,32 +61,28 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
 
             /* Ensure that the path exists and points into the Nix
                store. */
+            // FIXME: should we disallow products referring to other
+            // store paths, or that are outside the input closure?
             if (product.path == "" || product.path[0] != '/') continue;
-            try {
-                product.path = canonPath(product.path, true);
-            } catch (Error & e) { continue; }
-            if (!isInStore(product.path) || !pathExists(product.path)) continue;
+            product.path = canonPath(product.path);
+            if (!isInStore(product.path)) continue;
 
-            /*  FIXME: check that the path is in the input closure
-                of the build? */
+            auto st = accessor->stat(product.path);
+            if (st.type == FSAccessor::Type::tMissing) continue;
 
             product.name = product.path == output ? "" : baseNameOf(product.path);
 
-            struct stat st;
-            if (stat(product.path.c_str(), &st))
-                throw SysError(format("getting status of ‘%1%’") % product.path);
-
-            if (S_ISREG(st.st_mode)) {
+            if (st.type == FSAccessor::Type::tRegular) {
                 product.isRegular = true;
-                product.fileSize = st.st_size;
-                product.sha1hash = hashFile(htSHA1, product.path);
-                product.sha256hash = hashFile(htSHA256, product.path);
+                product.fileSize = st.fileSize;
+                auto contents = accessor->readFile(product.path);
+                product.sha1hash = hashString(htSHA1, contents);
+                product.sha256hash = hashString(htSHA256, contents);
             }
 
             res.products.push_back(product);
         }
     }
-#endif
 
     /* If no build products were explicitly declared, then add all
        outputs as a product of type "nix-build". */
@@ -110,31 +94,29 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
             product.subtype = output.first == "out" ? "" : output.first;
             product.name = storePathToName(product.path);
 
-#if 0
-            struct stat st;
-            if (stat(product.path.c_str(), &st))
-                throw SysError(format("getting status of ‘%1%’") % product.path);
-            if (S_ISDIR(st.st_mode))
-#endif
+            auto st = accessor->stat(product.path);
+            if (st.type == FSAccessor::Type::tMissing)
+                throw Error(format("getting status of ‘%1%’") % product.path);
+            if (st.type == FSAccessor::Type::tDirectory)
                 res.products.push_back(product);
         }
     }
 
-#if 0
     /* Get the release name from $output/nix-support/hydra-release-name. */
     for (auto & output : outputs) {
         Path p = output + "/nix-support/hydra-release-name";
-        if (!pathExists(p)) continue;
+        if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
         try {
-            res.releaseName = trim(readFile(p));
+            res.releaseName = trim(accessor->readFile(p));
         } catch (Error & e) { continue; }
         // FIXME: validate release name
     }
 
     /* Get metrics. */
     for (auto & output : outputs) {
-        auto file = secureRead(output + "/nix-support/hydra-metrics");
-        for (auto & line : tokenizeString<Strings>(std::get<1>(file), "\n")) {
+        Path metricsFile = output + "/nix-support/hydra-metrics";
+        if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
+        for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
             auto fields = tokenizeString<std::vector<std::string>>(line);
             if (fields.size() < 2) continue;
             BuildMetric metric;
@@ -144,7 +126,6 @@ BuildOutput getBuildOutput(nix::ref<Store> store, const Derivation & drv)
             res.metrics[metric.name] = metric;
         }
     }
-#endif
 
     return res;
 }
diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh
index 7fdd659b..72e8b4df 100644
--- a/src/hydra-queue-runner/build-result.hh
+++ b/src/hydra-queue-runner/build-result.hh
@@ -4,6 +4,7 @@
 
 #include "hash.hh"
 #include "derivations.hh"
+#include "store-api.hh"
 
 struct BuildProduct
 {
@@ -37,4 +38,5 @@ struct BuildOutput
     std::map<std::string, BuildMetric> metrics;
 };
 
-BuildOutput getBuildOutput(nix::ref<nix::Store> store, const nix::Derivation & drv);
+BuildOutput getBuildOutput(nix::ref<nix::Store> store,
+    nix::ref<nix::FSAccessor> accessor, const nix::Derivation & drv);
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 487a96b1..79566c4c 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -126,7 +126,7 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
             result.errorMsg = e.msg();
         }
 
-        if (result.success()) res = getBuildOutput(destStore, step->drv);
+        if (result.success()) res = getBuildOutput(destStore, destStore->getFSAccessor(), step->drv);
     }
 
     time_t stepStopTime = time(0);
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index f7e40827..b44e0836 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -159,7 +159,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
            all valid. So we mark this as a finished, cached build. */
         if (!step) {
             Derivation drv = readDerivation(build->drvPath);
-            BuildOutput res = getBuildOutput(destStore, drv);
+            BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv);
 
             pqxx::work txn(conn);
             time_t now = time(0);

From 6d741d2ffa77e3af7ce81cdc1e26a93c2e9b6cdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 15:21:44 +0100
Subject: [PATCH 25/40] Prevent download of NARs we just uploaded

---
 src/hydra-queue-runner/build-remote.cc | 9 ++++++---
 src/hydra-queue-runner/builder.cc      | 3 ++-
 src/hydra-queue-runner/state.hh        | 1 +
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index e6be7b10..45ebd695 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -116,11 +116,12 @@ static void copyClosureTo(ref<Store> destStore,
 
 
 static void copyClosureFrom(ref<Store> destStore,
-    FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived)
+    FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived,
+    std::shared_ptr<FSAccessor> accessor)
 {
     to << cmdExportPaths << 0 << paths;
     to.flush();
-    destStore->importPaths(false, from);
+    destStore->importPaths(false, from, accessor);
 
     for (auto & p : paths)
         bytesReceived += destStore->queryPathInfo(p).narSize;
@@ -297,9 +298,11 @@ void State::buildRemote(ref<Store> destStore,
             outputs.insert(output.second.path);
         MaintainCount mc(nrStepsCopyingFrom);
 
+        result.accessor = destStore->getFSAccessor();
+
         auto now1 = std::chrono::steady_clock::now();
 
-        copyClosureFrom(destStore, from, to, outputs, bytesReceived);
+        copyClosureFrom(destStore, from, to, outputs, bytesReceived, result.accessor);
 
         auto now2 = std::chrono::steady_clock::now();
 
diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index 79566c4c..b8cb3c87 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -126,7 +126,8 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
             result.errorMsg = e.msg();
         }
 
-        if (result.success()) res = getBuildOutput(destStore, destStore->getFSAccessor(), step->drv);
+        if (result.success())
+            res = getBuildOutput(destStore, ref<FSAccessor>(result.accessor), step->drv);
     }
 
     time_t stepStopTime = time(0);
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index 01dfd650..c1a5cfb6 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -53,6 +53,7 @@ struct RemoteResult : nix::BuildResult
     time_t startTime = 0, stopTime = 0;
     unsigned int overhead = 0;
     nix::Path logFile;
+    std::shared_ptr<nix::FSAccessor> accessor;
 
     bool canRetry()
     {

From b9afaadfb3e657552b69274970822f1106def0d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 16:16:36 +0100
Subject: [PATCH 26/40] Keep better bytesReceived/bytesSent stats

---
 src/hydra-queue-runner/build-remote.cc | 29 +++++++++-----------------
 src/hydra-queue-runner/finally.hh      | 12 +++++++++++
 2 files changed, 22 insertions(+), 19 deletions(-)
 create mode 100644 src/hydra-queue-runner/finally.hh

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 45ebd695..d0bf7d2c 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -8,6 +8,7 @@
 #include "state.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
+#include "finally.hh"
 
 using namespace nix;
 
@@ -75,7 +76,6 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil
 
 static void copyClosureTo(ref<Store> destStore,
     FdSource & from, FdSink & to, const PathSet & paths,
-    counter & bytesSent,
     bool useSubstitutes = false)
 {
     PathSet closure;
@@ -103,9 +103,6 @@ static void copyClosureTo(ref<Store> destStore,
 
     printMsg(lvlDebug, format("sending %1% missing paths") % missing.size());
 
-    for (auto & p : missing)
-        bytesSent += destStore->queryPathInfo(p).narSize;
-
     to << cmdImportPaths;
     destStore->exportPaths(missing, false, to);
     to.flush();
@@ -115,19 +112,6 @@ static void copyClosureTo(ref<Store> destStore,
 }
 
 
-static void copyClosureFrom(ref<Store> destStore,
-    FdSource & from, FdSink & to, const PathSet & paths, counter & bytesReceived,
-    std::shared_ptr<FSAccessor> accessor)
-{
-    to << cmdExportPaths << 0 << paths;
-    to.flush();
-    destStore->importPaths(false, from, accessor);
-
-    for (auto & p : paths)
-        bytesReceived += destStore->queryPathInfo(p).narSize;
-}
-
-
 void State::buildRemote(ref<Store> destStore,
     Machine::ptr machine, Step::ptr step,
     unsigned int maxSilentTime, unsigned int buildTimeout,
@@ -153,6 +137,11 @@ void State::buildRemote(ref<Store> destStore,
     FdSource from(child.from);
     FdSink to(child.to);
 
+    Finally updateStats([&]() {
+        bytesReceived += from.read;
+        bytesSent += to.written;
+    });
+
     /* Handshake. */
     bool sendDerivation = true;
     unsigned int remoteVersion;
@@ -239,7 +228,7 @@ void State::buildRemote(ref<Store> destStore,
 
         auto now1 = std::chrono::steady_clock::now();
 
-        copyClosureTo(destStore, from, to, inputs, bytesSent, true);
+        copyClosureTo(destStore, from, to, inputs, true);
 
         auto now2 = std::chrono::steady_clock::now();
 
@@ -302,7 +291,9 @@ void State::buildRemote(ref<Store> destStore,
 
         auto now1 = std::chrono::steady_clock::now();
 
-        copyClosureFrom(destStore, from, to, outputs, bytesReceived, result.accessor);
+        to << cmdExportPaths << 0 << outputs;
+        to.flush();
+        destStore->importPaths(false, from, result.accessor);
 
         auto now2 = std::chrono::steady_clock::now();
 
diff --git a/src/hydra-queue-runner/finally.hh b/src/hydra-queue-runner/finally.hh
new file mode 100644
index 00000000..47c64dea
--- /dev/null
+++ b/src/hydra-queue-runner/finally.hh
@@ -0,0 +1,12 @@
+#pragma once
+
+/* A trivial class to run a function at the end of a scope. */
+class Finally
+{
+private:
+    std::function<void()> fun;
+
+public:
+    Finally(std::function<void()> fun) : fun(fun) { }
+    ~Finally() { fun(); }
+};

From 9de336de7c45d056a9dc0669347f8ce04b32749f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 17:27:30 +0100
Subject: [PATCH 27/40] Proxy local binary caches via hydra-server

---
 src/lib/Hydra/Controller/Root.pm | 94 +++++++++++++++++++++++---------
 1 file changed, 68 insertions(+), 26 deletions(-)

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 48319479..8827d4f0 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -257,48 +257,90 @@ sub serialize : ActionClass('Serialize') { }
 sub nar :Local :Args(1) {
     my ($self, $c, $path) = @_;
 
-    $path = ($ENV{NIX_STORE_DIR} || "/nix/store")."/$path";
+    die if $path =~ /\//;
 
-    gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+    my $storeMode = $c->config->{store_mode} // "direct";
 
-    $c->stash->{current_view} = 'NixNAR';
-    $c->stash->{storePath} = $path;
+    if ($storeMode eq "s3-binary-cache") {
+        notFound($c, "There is no binary cache here.");
+    }
+
+    elsif ($storeMode eq "local-binary-cache") {
+        my $dir = $c->config->{binary_cache_dir};
+        $c->serve_static_file($dir . "/nar/" . $path);
+    }
+
+    else {
+        $path = $Nix::Config::storeDir . "/$path";
+
+        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+
+        $c->stash->{current_view} = 'NixNAR';
+        $c->stash->{storePath} = $path;
+    }
 }
 
 
 sub nix_cache_info :Path('nix-cache-info') :Args(0) {
     my ($self, $c) = @_;
-    $c->response->content_type('text/plain');
-    $c->stash->{plain}->{data} =
-        #"StoreDir: $Nix::Config::storeDir\n" . # FIXME
-        "StoreDir: " . ($ENV{NIX_STORE_DIR} || "/nix/store") . "\n" .
-        "WantMassQuery: 0\n" .
-        # Give Hydra binary caches a very low priority (lower than the
-        # static binary cache http://nixos.org/binary-cache).
-        "Priority: 100\n";
-    setCacheHeaders($c, 24 * 60 * 60);
-    $c->forward('Hydra::View::Plain');
+
+    my $storeMode = $c->config->{store_mode} // "direct";
+
+    if ($storeMode eq "s3-binary-cache") {
+        notFound($c, "There is no binary cache here.");
+    }
+
+    elsif ($storeMode eq "local-binary-cache") {
+        my $dir = $c->config->{binary_cache_dir};
+        $c->serve_static_file($dir . "/nix-cache-info");
+    }
+
+    else {
+        $c->response->content_type('text/plain');
+        $c->stash->{plain}->{data} =
+            "StoreDir: $Nix::Config::storeDir\n" .
+            "WantMassQuery: 0\n" .
+            # Give Hydra binary caches a very low priority (lower than the
+            # static binary cache http://nixos.org/binary-cache).
+            "Priority: 100\n";
+        setCacheHeaders($c, 24 * 60 * 60);
+        $c->forward('Hydra::View::Plain');
+    }
 }
 
 
 sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) {
     my ($self, $c) = @_;
-    my $hash = $c->req->captures->[0];
 
-    die if length($hash) != 32;
-    my $path = queryPathFromHashPart($hash);
+    my $storeMode = $c->config->{store_mode} // "direct";
 
-    if (!$path) {
-        $c->response->status(404);
-        $c->response->content_type('text/plain');
-        $c->stash->{plain}->{data} = "does not exist\n";
-        $c->forward('Hydra::View::Plain');
-        setCacheHeaders($c, 60 * 60);
-        return;
+    if ($storeMode eq "s3-binary-cache") {
+        notFound($c, "There is no binary cache here.");
     }
 
-    $c->stash->{storePath} = $path;
-    $c->forward('Hydra::View::NARInfo');
+    elsif ($storeMode eq "local-binary-cache") {
+        my $dir = $c->config->{binary_cache_dir};
+        $c->serve_static_file($dir . "/" . $c->req->captures->[0] . ".narinfo");
+    }
+
+    else {
+        my $hash = $c->req->captures->[0];
+
+        die if length($hash) != 32;
+        my $path = queryPathFromHashPart($hash);
+
+        if (!$path) {
+            $c->response->status(404);
+            $c->response->content_type('text/plain');
+            $c->stash->{plain}->{data} = "does not exist\n";
+            $c->forward('Hydra::View::Plain');
+            setCacheHeaders($c, 60 * 60);
+            return;
+        }
+
+        $c->stash->{storePath} = $path;
+        $c->forward('Hydra::View::NARInfo');
+    }
 }
 
 

From b00bdefa98bd86de7e66e6d8d33f1bc5da81330f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 17:28:16 +0100
Subject: [PATCH 28/40] Fix hydra-server signing

---
 src/lib/Hydra/View/NARInfo.pm | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm
index d04b9e09..4afe4041 100644
--- a/src/lib/Hydra/View/NARInfo.pm
+++ b/src/lib/Hydra/View/NARInfo.pm
@@ -36,13 +36,10 @@ sub process {
     # Optionally, sign the NAR info file we just created.
     my $secretKeyFile = $c->config->{binary_cache_secret_key_file};
     if (defined $secretKeyFile) {
-        my $s = readFile $secretKeyFile;
-        chomp $s;
-        my ($keyName, $secretKey) = split ":", $s;
-        die "invalid secret key file\n" unless defined $keyName && defined $secretKey;
+        my $secretKey = readFile $secretKeyFile;
         my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
-        my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), "");
-        $info .= "Sig: $keyName:$sig\n";
+        my $sig = signString($secretKey, $fingerprint);
+        $info .= "Sig: $sig\n";
     }
 
     setCacheHeaders($c, 24 * 60 * 60);

From 07e5fc5618b3944036d90f014ede2ca708f53aab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 17:28:26 +0100
Subject: [PATCH 29/40] Hackery to make downloads work when using a binary
 cache

---
 src/lib/Hydra/Controller/Build.pm | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 72117a47..23535922 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -64,7 +64,10 @@ sub build_GET {
     my $build = $c->stash->{build};
 
     $c->stash->{template} = 'build.tt';
-    $c->stash->{available} = all { isValidPath($_->path) } $build->buildoutputs->all;
+    $c->stash->{available} =
+        ($c->config->{store_mode} // "direct") eq "direct"
+        ? all { isValidPath($_->path) } $build->buildoutputs->all
+        : 1; # FIXME
     $c->stash->{drvAvailable} = isValidPath $build->drvpath;
 
     if ($build->finished && $build->iscachedbuild) {
@@ -219,7 +222,31 @@ sub download : Chained('buildChain') PathPart {
     }
     notFound($c, "Build doesn't have a product $productRef.") if !defined $product;
 
-    notFound($c, "Build product " . $product->path . " has disappeared.") unless -e $product->path;
+    if ($product->path !~ /^($Nix::Config::storeDir\/[^\/]+)/) {
+        die "Invalid store path " . $product->path . ".\n";
+    }
+    my $storePath = $1;
+
+    # Hack to get downloads to work on binary cache stores: if the
+    # store path is not available locally, then import it into the
+    # local store. FIXME: find a better way; this can require an
+    # unbounded amount of space.
+    if (!isValidPath($storePath)) {
+        my $storeMode = $c->config->{store_mode} // "direct";
+        notFound($c, "File " . $product->path . " has disappeared.")
+            if $storeMode eq "direct";
+        my $url =
+            $storeMode eq "local-binary-cache" ? "file://" . $c->config->{binary_cache_dir} :
+            $storeMode eq "s3-binary-cache" ? "https://" . $c->config->{binary_cache_s3_bucket} . ".s3.amazonaws.com/" :
+            die;
+        my $args =
+            defined $c->config->{binary_cache_public_key_file}
+            ? "--option binary-cache-public-keys " . read_file($c->config->{binary_cache_public_key_file}) . "\n"
+            : "";
+        system("nix-store --realise '$storePath' --option extra-binary-caches '$url' $args>/dev/null");
+    }
+
+    notFound($c, "File " . $product->path . " does not exist.") unless -e $product->path;
 
     return $c->res->redirect(defaultUriForProduct($self, $c, $product, @path))
         if scalar @path == 0 && ($product->name || $product->defaultpath);

From b1ce76c2b405f91da938a811ff9be6e408a303ec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 19:54:32 +0100
Subject: [PATCH 30/40] Fix test

nix-support/failed is supposed to be a file, not a directory.
---
 tests/jobs/succeed-with-failed.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/jobs/succeed-with-failed.sh b/tests/jobs/succeed-with-failed.sh
index 51f79931..bd1214b9 100755
--- a/tests/jobs/succeed-with-failed.sh
+++ b/tests/jobs/succeed-with-failed.sh
@@ -1,3 +1,3 @@
 #! /bin/sh
-mkdir -p $out/nix-support/failed
-
+mkdir -p $out/nix-support
+touch $out/nix-support/failed

From c635f5d0ea6ff9ca3089c3749b9bd2dcdd1c5289 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 19:54:55 +0100
Subject: [PATCH 31/40] Fix Makefile.am

---
 src/hydra-queue-runner/Makefile.am | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am
index 2cf4fcdb..12b7e01e 100644
--- a/src/hydra-queue-runner/Makefile.am
+++ b/src/hydra-queue-runner/Makefile.am
@@ -3,7 +3,8 @@ bin_PROGRAMS = hydra-queue-runner
 hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
  builder.cc build-result.cc build-remote.cc  \
  build-result.hh counter.hh token-server.hh state.hh db.hh \
- s3-binary-cache-store.hh s3-binary-cache-store.cc
+ s3-binary-cache-store.hh s3-binary-cache-store.cc \
+ finally.hh
 hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
 
 AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3

From 53ca41ef9f200248fafa171fb429b9e5028495a7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 20:57:47 +0100
Subject: [PATCH 32/40] Use US standard S3 region

---
 src/hydra-queue-runner/s3-binary-cache-store.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 4a78033a..4484d2af 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -44,7 +44,7 @@ S3BinaryCacheStore::S3BinaryCacheStore(std::shared_ptr<Store> localStore,
 ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()
 {
     auto res = make_ref<Aws::Client::ClientConfiguration>();
-    res->region = Aws::Region::EU_WEST_1;
+    res->region = Aws::Region::US_EAST_1;
     res->requestTimeoutMs = 600 * 1000;
     return res;
 }
@@ -66,8 +66,8 @@ void S3BinaryCacheStore::init()
             .WithBucket(bucketName)
             .WithCreateBucketConfiguration(
                 Aws::S3::Model::CreateBucketConfiguration()
-                .WithLocationConstraint(
-                    Aws::S3::Model::BucketLocationConstraint::eu_west_1))));
+                /* .WithLocationConstraint(
+                    Aws::S3::Model::BucketLocationConstraint::US) */ )));
     }
 
     BinaryCacheStore::init();

From e8cdfe5171872e73471de73494941117604d9611 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 21:14:40 +0100
Subject: [PATCH 33/40] hydra-server: Don't barf if the binary cache public key
 can't be read

---
 src/lib/Hydra/Controller/Build.pm | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index 23535922..cdf68456 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -239,10 +239,12 @@ sub download : Chained('buildChain') PathPart {
             $storeMode eq "local-binary-cache" ? "file://" . $c->config->{binary_cache_dir} :
             $storeMode eq "s3-binary-cache" ? "https://" . $c->config->{binary_cache_s3_bucket} . ".s3.amazonaws.com/" :
             die;
-        my $args =
-            defined $c->config->{binary_cache_public_key_file}
-            ? "--option binary-cache-public-keys " . read_file($c->config->{binary_cache_public_key_file}) . "\n"
-            : "";
+        my $args = "";
+        if (defined $c->config->{binary_cache_public_key_file}
+            && -r $c->config->{binary_cache_public_key_file})
+        {
+            $args = "--option binary-cache-public-keys " . read_file($c->config->{binary_cache_public_key_file});
+        }
         system("nix-store --realise '$storePath' --option extra-binary-caches '$url' $args>/dev/null");
     }
 

From 6bb860fd6e6de3ab360905dc527f8db7045f3cf5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 21:15:05 +0100
Subject: [PATCH 34/40] Add FIXME

---
 src/hydra-queue-runner/build-remote.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index d0bf7d2c..039c2e2a 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -86,6 +86,7 @@ static void copyClosureTo(ref<Store> destStore,
        enabled. This prevents a race where the remote host
        garbage-collect paths that are already there. Optionally, ask
        the remote host to substitute missing paths. */
+    // FIXME: substitute output pollutes our build log
     to << cmdQueryValidPaths << 1 << useSubstitutes << closure;
     to.flush();
 

From 1a055e7e9e20488463434b80d2d929e4df802c60 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 21:31:08 +0100
Subject: [PATCH 35/40] Reduce severity level of some message

---
 src/hydra-queue-runner/s3-binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 4484d2af..9fb39aad 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -135,7 +135,7 @@ void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
 
-    printMsg(lvlError, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
+    printMsg(lvlInfo, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
         % bucketName % path % data.size() % duration);
 
     stats.putTimeMs += duration;
@@ -164,7 +164,7 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
 
-    printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
+    printMsg(lvlInfo, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
         % bucketName % path % res.size() % duration);
 
     stats.getBytes += res.size();

From 16933545061de240ee1f686a092f336c684dd2a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 21:45:59 +0100
Subject: [PATCH 36/40] Remove unnecessary call to hydra-queue-runner --unlock

---
 hydra-module.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/hydra-module.nix b/hydra-module.nix
index 30082e0f..1e491ba0 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -291,8 +291,7 @@ in
           IN_SYSTEMD = "1"; # to get log severity levels
         };
         serviceConfig =
-          { ExecStartPre = "${cfg.package}/bin/hydra-queue-runner --unlock";
-            ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v";
+          { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes false";
             ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
             User = "hydra-queue-runner";
             Restart = "always";

From 610a8d67ae151da69f0113d058e2a6521ba400a4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 26 Feb 2016 22:40:27 +0100
Subject: [PATCH 37/40] Better AWS error messages

---
 .../s3-binary-cache-store.cc                  | 31 ++++++++++---------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc
index 9fb39aad..146c26fc 100644
--- a/src/hydra-queue-runner/s3-binary-cache-store.cc
+++ b/src/hydra-queue-runner/s3-binary-cache-store.cc
@@ -22,12 +22,12 @@ struct S3Error : public Error
 /* Helper: given an Outcome<R, E>, return R in case of success, or
    throw an exception in case of an error. */
 template<typename R, typename E>
-R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
+R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
 {
     if (!outcome.IsSuccess())
         throw S3Error(
             outcome.GetError().GetErrorType(),
-            format("AWS error: %1%") % outcome.GetError().GetMessage());
+            fs.s + ": " + outcome.GetError().GetMessage());
     return outcome.GetResultWithOwnership();
 }
 
@@ -59,15 +59,16 @@ void S3BinaryCacheStore::init()
 
     if (!res.IsSuccess()) {
         if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
-            throw Error(format("AWS error: %1%") % res.GetError().GetMessage());
+            throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
 
-        checkAws(client->CreateBucket(
-            Aws::S3::Model::CreateBucketRequest()
-            .WithBucket(bucketName)
-            .WithCreateBucketConfiguration(
-                Aws::S3::Model::CreateBucketConfiguration()
-                /* .WithLocationConstraint(
-                    Aws::S3::Model::BucketLocationConstraint::US) */ )));
+        checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
+            client->CreateBucket(
+                Aws::S3::Model::CreateBucketRequest()
+                .WithBucket(bucketName)
+                .WithCreateBucketConfiguration(
+                    Aws::S3::Model::CreateBucketConfiguration()
+                    /* .WithLocationConstraint(
+                       Aws::S3::Model::BucketLocationConstraint::US) */ )));
     }
 
     BinaryCacheStore::init();
@@ -107,7 +108,7 @@ bool S3BinaryCacheStore::fileExists(const std::string & path)
         if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
             && error.GetMessage().find("404") != std::string::npos)
             return false;
-        throw Error(format("AWS error: %1%") % error.GetMessage());
+        throw Error(format("AWS error fetching ‘%s’") % path % error.GetMessage());
     }
 
     return true;
@@ -129,7 +130,8 @@ void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string
 
     auto now1 = std::chrono::steady_clock::now();
 
-    auto result = checkAws(client->PutObject(request));
+    auto result = checkAws(format("AWS error uploading ‘%s’") % path,
+        client->PutObject(request));
 
     auto now2 = std::chrono::steady_clock::now();
 
@@ -156,7 +158,8 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
 
     auto now1 = std::chrono::steady_clock::now();
 
-    auto result = checkAws(client->GetObject(request));
+    auto result = checkAws(format("AWS error fetching ‘%s’") % path,
+        client->GetObject(request));
 
     auto now2 = std::chrono::steady_clock::now();
 
@@ -164,7 +167,7 @@ std::string S3BinaryCacheStore::getFile(const std::string & path)
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
 
-    printMsg(lvlInfo, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
+    printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
         % bucketName % path % res.size() % duration);
 
     stats.getBytes += res.size();

From ad035b52271cb374ff6ba699f521d75dc9ae66a4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Sun, 28 Feb 2016 14:09:04 +0100
Subject: [PATCH 38/40] hydra-queue-runner: Enable core dumps

---
 hydra-module.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hydra-module.nix b/hydra-module.nix
index 1e491ba0..059769fa 100644
--- a/hydra-module.nix
+++ b/hydra-module.nix
@@ -295,6 +295,10 @@ in
             ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
             User = "hydra-queue-runner";
             Restart = "always";
+
+            # Ensure we can get core dumps.
+            LimitCORE = "infinity";
+            WorkingDirectory = "${baseDir}/queue-runner";
           };
       };
 

From 922dc541c26a619d1073244bfa63525bbe62f260 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 29 Feb 2016 11:58:06 +0100
Subject: [PATCH 39/40] Add log message

---
 src/hydra-queue-runner/build-remote.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc
index 039c2e2a..ef3ca2e3 100644
--- a/src/hydra-queue-runner/build-remote.cc
+++ b/src/hydra-queue-runner/build-remote.cc
@@ -276,6 +276,7 @@ void State::buildRemote(ref<Store> destStore,
     /* If the path was substituted or already valid, then we didn't
        get a build log. */
     if (result.status == BuildResult::Substituted || result.status == BuildResult::AlreadyValid) {
+        printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName);
         unlink(result.logFile.c_str());
         result.logFile = "";
     }

From 7cd08c7c46621faed76bfb11fa3775d02a42eb0c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 29 Feb 2016 15:10:30 +0100
Subject: [PATCH 40/40] Warn if PostgreSQL appears stalled

---
 src/hydra-queue-runner/builder.cc            |  6 ++++++
 src/hydra-queue-runner/counter.hh            |  5 +++++
 src/hydra-queue-runner/hydra-queue-runner.cc | 12 ++++++++++++
 src/hydra-queue-runner/queue-monitor.cc      |  6 ++++++
 src/hydra-queue-runner/state.hh              |  3 +++
 5 files changed, 32 insertions(+)

diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc
index b8cb3c87..9179fe42 100644
--- a/src/hydra-queue-runner/builder.cc
+++ b/src/hydra-queue-runner/builder.cc
@@ -112,6 +112,7 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
         /* Create a build step record indicating that we started
            building. */
         {
+            auto mc = startDbUpdate();
             pqxx::work txn(*conn);
             stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bssBusy);
             txn.commit();
@@ -165,6 +166,7 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
             retry = step_->tries + 1 < maxTries;
         }
         if (retry) {
+            auto mc = startDbUpdate();
             pqxx::work txn(*conn);
             finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, build->id,
                 stepNr, machine->sshName, bssAborted, result.errorMsg);
@@ -213,6 +215,8 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
 
             /* Update the database. */
             {
+                auto mc = startDbUpdate();
+
                 pqxx::work txn(*conn);
 
                 finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
@@ -299,6 +303,8 @@ bool State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
 
             /* Update the database. */
             {
+                auto mc = startDbUpdate();
+
                 pqxx::work txn(*conn);
 
                 BuildStatus buildStatus =
diff --git a/src/hydra-queue-runner/counter.hh b/src/hydra-queue-runner/counter.hh
index 1943d1c3..6afff99d 100644
--- a/src/hydra-queue-runner/counter.hh
+++ b/src/hydra-queue-runner/counter.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <atomic>
+#include <functional>
 
 typedef std::atomic<unsigned long> counter;
 
@@ -8,5 +9,9 @@ struct MaintainCount
 {
     counter & c;
     MaintainCount(counter & c) : c(c) { c++; }
+    MaintainCount(counter & c, std::function<void(unsigned long)> warn) : c(c)
+    {
+        warn(++c);
+    }
     ~MaintainCount() { auto prev = c--; assert(prev); }
 };
diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc
index 2e4229c2..d7e80f82 100644
--- a/src/hydra-queue-runner/hydra-queue-runner.cc
+++ b/src/hydra-queue-runner/hydra-queue-runner.cc
@@ -45,6 +45,16 @@ State::State()
 }
 
 
+MaintainCount State::startDbUpdate()
+{
+    return MaintainCount(nrActiveDbUpdates, [](unsigned long c) {
+        if (c > 6) {
+            printMsg(lvlError, format("warning: %d concurrent database updates; PostgreSQL may be stalled") % c);
+        }
+    });
+}
+
+
 ref<Store> State::getLocalStore()
 {
     return ref<Store>(_localStore);
@@ -552,6 +562,7 @@ void State::dumpStatus(Connection & conn, bool log)
         root.attr("nrQueueWakeups", nrQueueWakeups);
         root.attr("nrDispatcherWakeups", nrDispatcherWakeups);
         root.attr("nrDbConnections", dbPool.count());
+        root.attr("nrActiveDbUpdates", nrActiveDbUpdates);
         {
             root.attr("machines");
             JSONObject nested(out);
@@ -661,6 +672,7 @@ void State::dumpStatus(Connection & conn, bool log)
     if (log) printMsg(lvlInfo, format("status: %1%") % out.str());
 
     {
+        auto mc = startDbUpdate();
         pqxx::work txn(conn);
         // FIXME: use PostgreSQL 9.5 upsert.
         txn.exec("delete from SystemStatus where what = 'queue-runner'");
diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc
index b44e0836..c9ea6da2 100644
--- a/src/hydra-queue-runner/queue-monitor.cc
+++ b/src/hydra-queue-runner/queue-monitor.cc
@@ -124,6 +124,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
             /* Derivation has been GC'ed prematurely. */
             printMsg(lvlError, format("aborting GC'ed build %1%") % build->id);
             if (!build->finishedInDB) {
+                auto mc = startDbUpdate();
                 pqxx::work txn(conn);
                 txn.parameterized
                     ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
@@ -161,10 +162,13 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
             Derivation drv = readDerivation(build->drvPath);
             BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv);
 
+            {
+            auto mc = startDbUpdate();
             pqxx::work txn(conn);
             time_t now = time(0);
             markSucceededBuild(txn, build, res, true, now, now);
             txn.commit();
+            }
 
             build->finishedInDB = true;
 
@@ -178,6 +182,7 @@ bool State::getQueuedBuilds(Connection & conn, ref<Store> localStore,
             if (checkCachedFailure(r, conn)) {
                 printMsg(lvlError, format("marking build %1% as cached failure") % build->id);
                 if (!build->finishedInDB) {
+                    auto mc = startDbUpdate();
                     pqxx::work txn(conn);
 
                     /* Find the previous build step record, first by
@@ -421,6 +426,7 @@ Step::ptr State::createStep(ref<Store> destStore,
                     time_t stopTime = time(0);
 
                     {
+                        auto mc = startDbUpdate();
                         pqxx::work txn(conn);
                         createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", i.second.path);
                         txn.commit();
diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh
index c1a5cfb6..be3ead24 100644
--- a/src/hydra-queue-runner/state.hh
+++ b/src/hydra-queue-runner/state.hh
@@ -313,6 +313,7 @@ private:
     counter nrDispatcherWakeups{0};
     counter bytesSent{0};
     counter bytesReceived{0};
+    counter nrActiveDbUpdates{0};
 
     /* Log compressor work queue. */
     nix::Sync<std::queue<nix::Path>> logCompressorQueue;
@@ -359,6 +360,8 @@ public:
 
 private:
 
+    MaintainCount startDbUpdate();
+
     /* Return a store object that can access derivations produced by
        hydra-evaluator. */
     nix::ref<nix::Store> getLocalStore();