Use a single BinaryCacheStore for all threads

This will make it easier to do caching / keep stats. Also, we won't
have S3Client's connection pooling if we create multiple S3Client
instances.

src/hydra-queue-runner/binary-cache-store.cc

@@ -11,9 +11,9 @@
 
 namespace nix {
 
-BinaryCacheStore::BinaryCacheStore(ref<Store> localStore,
+BinaryCacheStore::BinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile)
-    : localStore(localStore)
+    : storeFactory(storeFactory)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@@ -192,6 +192,8 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     PathSet left;
 
+    auto localStore = storeFactory();
+
     for (auto & storePath : paths) {
         if (!localStore->isValidPath(storePath)) {
             left.insert(storePath);
@@ -210,6 +212,8 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
 
 void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
+    auto localStore = storeFactory();
+
     for (auto & storePath : paths) {
         assert(!isDerivation(storePath));
 

src/hydra-queue-runner/binary-cache-store.hh

@@ -7,18 +7,23 @@ namespace nix {
 
 struct NarInfo;
 
+/* While BinaryCacheStore is thread-safe, LocalStore and RemoteStore
+   aren't. Until they are, use a factory to produce a thread-local
+   local store. */
+typedef std::function<ref<Store>()> StoreFactory;
+
 class BinaryCacheStore : public Store
 {
 private:
 
-    ref<Store> localStore;
-
     std::unique_ptr<SecretKey> secretKey;
     std::unique_ptr<PublicKeys> publicKeys;
 
+    StoreFactory storeFactory;
+
 protected:
 
-    BinaryCacheStore(ref<Store> localStore,
+    BinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile);
 
     virtual bool fileExists(const std::string & path) = 0;

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -23,6 +23,21 @@ State::State()
     if (hydraData == "") throw Error("$HYDRA_DATA must be set");
 
     logDir = canonPath(hydraData + "/build-logs");
+
+#if 0
+    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "/tmp/binary-cache");
+#endif
+
+    auto store = std::make_shared<S3BinaryCacheStore>(
+        []() { return openStore(); },
+        "/home/eelco/Misc/Keys/test.nixos.org/secret",
+        "/home/eelco/Misc/Keys/test.nixos.org/public",
+        "nix-test-cache-3");;
+    store->init();
+    _destStore = store;
 }
 
 
@@ -34,18 +49,7 @@ ref<Store> State::getLocalStore()
 
 ref<Store> State::getDestStore()
 {
-#if 0
+    return ref<Store>(_destStore);
-    auto store = make_ref<LocalBinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "/tmp/binary-cache");
-#endif
-    auto store = make_ref<S3BinaryCacheStore>(getLocalStore(),
-        "/home/eelco/Misc/Keys/test.nixos.org/secret",
-        "/home/eelco/Misc/Keys/test.nixos.org/public",
-        "nix-test-cache-3");
-    store->init();
-    return store;
 }
 
 

src/hydra-queue-runner/local-binary-cache-store.cc

@@ -2,10 +2,10 @@
 
 namespace nix {
 
-LocalBinaryCacheStore::LocalBinaryCacheStore(ref<Store> localStore,
+LocalBinaryCacheStore::LocalBinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const Path & binaryCacheDir)
-    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
     , binaryCacheDir(binaryCacheDir)
 {
 }

src/hydra-queue-runner/local-binary-cache-store.hh

@@ -12,7 +12,7 @@ private:
 
 public:
 
-    LocalBinaryCacheStore(ref<Store> localStore,
+    LocalBinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const Path & binaryCacheDir);
 

src/hydra-queue-runner/s3-binary-cache-store.cc

@@ -20,10 +20,10 @@ R && checkAws(Aws::Utils::Outcome<R, E> && outcome)
     return outcome.GetResultWithOwnership();
 }
 
-S3BinaryCacheStore::S3BinaryCacheStore(ref<Store> localStore,
+S3BinaryCacheStore::S3BinaryCacheStore(const StoreFactory & storeFactory,
     const Path & secretKeyFile, const Path & publicKeyFile,
     const std::string & bucketName)
-    : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+    : BinaryCacheStore(storeFactory, secretKeyFile, publicKeyFile)
     , bucketName(bucketName)
     , config(makeConfig())
     , client(make_ref<Aws::S3::S3Client>(*config))

src/hydra-queue-runner/s3-binary-cache-store.hh

@@ -18,7 +18,7 @@ private:
 
 public:
 
-    S3BinaryCacheStore(ref<Store> localStore,
+    S3BinaryCacheStore(const StoreFactory & storeFactory,
         const Path & secretKeyFile, const Path & publicKeyFile,
         const std::string & bucketName);
 

src/hydra-queue-runner/state.hh

@@ -346,6 +346,9 @@ private:
 
     std::atomic<time_t> lastDispatcherCheck{0};
 
+    /* Destination store. */
+    std::shared_ptr<nix::Store> _destStore;
+
 public:
     State();