From 79c169d1c615211af69c1cbc6218f7465a4f81ed Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 09:49:09 -0500
Subject: [PATCH 01/24] Allow substituting from different storeDir

Substituters can substitute from one store dir to another with a
little bit of help. The store api just needs to have a CA so it can
recompute the store path based on the new store dir. We can only do
this for fixed output derivations with no references, though.
---
 src/libstore/local-store.cc  | 16 ++++++++++++----
 src/libstore/local-store.hh  |  3 ++-
 src/libstore/misc.cc         | 27 ++++++++++++++++++++++++++-
 src/libstore/remote-store.cc |  2 +-
 src/libstore/remote-store.hh |  2 +-
 src/libstore/store-api.cc    |  7 +++++++
 src/libstore/store-api.hh    |  2 +-
 7 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 80851b591..6385453cc 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 
 
 void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
-        if (sub->storeDir != storeDir) continue;
-        for (auto & path : paths) {
-            if (infos.count(path)) continue;
+        for (auto & path_ : paths) {
+            auto path(path_.clone());
             debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
             try {
                 auto info = sub->queryPathInfo(path);
+
+                auto ca = pathsCA.find(printStorePath(path));
+                if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) {
+                    if (!hasPrefix(ca->second, "fixed:"))
+                        continue;
+                    // recompute store path so that we can use a fixed output ca
+                    path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name());
+                } else continue;
+
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
                 infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index c1e75390c..5a976e3e1 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -142,7 +142,8 @@ public:
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
     void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) override;
+        SubstitutablePathInfos & infos,
+        std::map<std::string, std::string> pathsCA = {}) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 9c47fe524..1b0a055d3 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
+static std::optional<std::string> getDerivationCA(ref<Derivation> drv)
+{
+    auto outputHashMode = drv->env.find("outputHashMode");
+    auto outputHashAlgo = drv->env.find("outputHashAlgo");
+    auto outputHash = drv->env.find("outputHash");
+    if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) {
+        auto ht = parseHashType(outputHashAlgo->second);
+        auto h = Hash(outputHash->second, ht);
+        FileIngestionMethod ingestionMethod;
+        if (outputHashMode->second == "recursive")
+            ingestionMethod = FileIngestionMethod::Recursive;
+        else if (outputHashMode->second == "flat")
+            ingestionMethod = FileIngestionMethod::Flat;
+        else
+            throw Error("unknown ingestion method: '%s'", outputHashMode->second);
+        return makeFixedOutputCA(ingestionMethod, h);
+    }
+
+    return std::nullopt;
+}
+
 void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
     unsigned long long & downloadSize_, unsigned long long & narSize_)
@@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         SubstitutablePathInfos infos;
         StorePathSet paths; // FIXME
         paths.insert(outPath.clone());
-        querySubstitutablePathInfos(paths, infos);
+
+        std::map<std::string, std::string> pathsCA = {};
+        if (auto ca = getDerivationCA(drv))
+            pathsCA.insert({outPathS, *ca});
+        querySubstitutablePathInfos(paths, infos, pathsCA);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5c36693e6..956980f49 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 
 
 void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
 {
     if (paths.empty()) return;
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 3c86b4524..867d650bc 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -58,7 +58,7 @@ public:
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
     void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) override;
+        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 095363d0c..40000da01 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -580,6 +580,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
+    // recompute store path on the chance dstStore does it differently
+    if (hasPrefix(info->ca, "fixed:") && info->references.empty()) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name());
+        info = info2;
+    }
+
     if (!info->narHash) {
         StringSink sink;
         srcStore->narFromPath({storePath}, sink);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b1e25fc7d..6f961329c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -445,7 +445,7 @@ public:
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) { return; };
+        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) { return; };
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,

From 3e3eaa90dd1aaf4684697f27726ec72aec75206c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 09:51:44 -0500
Subject: [PATCH 02/24] Remove hashed-mirrors

---
 doc/manual/command-ref/conf-file.xml | 28 ----------------------------
 src/libstore/builtins/fetchurl.cc    | 14 --------------
 src/libstore/globals.hh              |  3 ---
 3 files changed, 45 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..9c55526a3 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,34 +370,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
-
-    <listitem><para>A list of web servers used by
-    <function>builtins.fetchurl</function> to obtain files by
-    hash. The default is
-    <literal>http://tarballs.nixos.org/</literal>. Given a hash type
-    <replaceable>ht</replaceable> and a base-16 hash
-    <replaceable>h</replaceable>, Nix will try to download the file
-    from
-    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
-    This allows files to be downloaded even if they have disappeared
-    from their original URI. For example, given the default mirror
-    <literal>http://tarballs.nixos.org/</literal>, when building the derivation
-
-<programlisting>
-builtins.fetchurl {
-  url = "https://example.org/foo-1.2.3.tar.xz";
-  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
-
-    Nix will attempt to download this file from
-    <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
-    first. If it is not available there, if will try the original URI.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 486babf14..254cb4fce 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,20 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
-    /* Try the hashed mirrors first. */
-    if (getAttr("outputHashMode") == "flat")
-        for (auto hashedMirror : settings.hashedMirrors.get())
-            try {
-                if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
-                auto h = Hash(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
-                return;
-            } catch (Error & e) {
-                debug(e.what());
-            }
-
-    /* Otherwise try the specified URL. */
     fetch(mainUrl);
 }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index da95fd3ae..2e1e405b3 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -339,9 +339,6 @@ public:
         "setuid/setgid bits or with file capabilities."};
 #endif
 
-    Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
-        "A list of servers used by builtins.fetchurl to fetch files by hash."};
-
     Setting<uint64_t> minFree{this, 0, "min-free",
         "Automatically run the garbage collector when free disk space drops below the specified amount."};
 

From 11c97070f3d845a9313f137c41974c021429e834 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 10:14:03 -0500
Subject: [PATCH 03/24] Fix storeDir != storeDir condition

this needs to only continue if the path replacement fails.
---
 src/libstore/local-store.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 6385453cc..4cecf0992 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -866,12 +866,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
                 auto info = sub->queryPathInfo(path);
 
                 auto ca = pathsCA.find(printStorePath(path));
-                if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) {
+                if (info->references.empty() && ca != pathsCA.end()) {
                     if (!hasPrefix(ca->second, "fixed:"))
                         continue;
-                    // recompute store path so that we can use a fixed output ca
+                    // recompute store path so that we can use a different store path
                     path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name());
-                } else continue;
+                } else if (sub->storeDir != storeDir) continue;
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));

From 1c55f16a1648431301f044710644a674c018321d Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 12:26:08 -0500
Subject: [PATCH 04/24] Add --flat to nix add-to-store

This can be used to add flat hashes to the nix store.
---
 src/nix/add-to-store.cc | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f43f774c1..a5a77f17c 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
     std::optional<std::string> namePart;
+    FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
 
     CmdAddToStore()
     {
@@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             .labels = {"name"},
             .handler = {&namePart},
         });
+
+        addFlag({
+            .longName = "flat",
+            .shortName = 0,
+            .description = "use flat file ingestion",
+            .handler = {&ingestionMethod, FileIngestionMethod::Flat},
+        });
     }
 
     std::string description() override
@@ -45,10 +53,24 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         auto narHash = hashString(htSHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
+        ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart));
         info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
+
+        Hash hash;
+        switch (ingestionMethod) {
+        case FileIngestionMethod::Recursive: {
+            hash = info.narHash;
+            break;
+        }
+        case FileIngestionMethod::Flat: {
+            HashSink hsink(htSHA256);
+            readFile(path, hsink);
+            hash = hsink.finish().first;
+            break;
+        }
+        }
+        info.ca = makeFixedOutputCA(ingestionMethod, hash);
 
         if (!dryRun) {
             auto source = StringSource { *sink.s };

From 2f2ac850b54110a81e3338468161cf132618c156 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 14:03:17 -0500
Subject: [PATCH 05/24] Compute new store path correctly

---
 src/libstore/local-store.cc | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4cecf0992..bf005a3b3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -859,19 +859,28 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
-        for (auto & path_ : paths) {
-            auto path(path_.clone());
-            debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
-            try {
-                auto info = sub->queryPathInfo(path);
+        for (auto & path : paths) {
+            auto subPath(path.clone());
 
-                auto ca = pathsCA.find(printStorePath(path));
-                if (info->references.empty() && ca != pathsCA.end()) {
-                    if (!hasPrefix(ca->second, "fixed:"))
-                        continue;
-                    // recompute store path so that we can use a different store path
-                    path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name());
-                } else if (sub->storeDir != storeDir) continue;
+            auto ca_ = pathsCA.find(printStorePath(path));
+            // recompute store path so that we can use a different store root
+            if (ca_ != pathsCA.end()) {
+                auto ca(ca_->second);
+                if (!hasPrefix(ca, "fixed:"))
+                    continue;
+                FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 };
+                Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
+                subPath = makeFixedOutputPath(ingestionMethod, hash, path.name());
+                if (subPath != path)
+                    debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri());
+            } else if (sub->storeDir != storeDir) continue;
+
+            debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
+            try {
+                auto info = sub->queryPathInfo(subPath);
+
+                if (sub->storeDir != storeDir && !info->isContentAddressed(*sub))
+                    continue;
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));

From 9077b9e6b28b94cdfac80ea4afbe18d13a7b2de6 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 14:27:28 -0500
Subject: [PATCH 06/24] Separate dstStore path from srcStore path

---
 src/libstore/store-api.cc | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 40000da01..1b340bf6c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -649,15 +649,24 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
     processGraph<Path>(pool,
         PathSet(missing.begin(), missing.end()),
 
-        [&](const Path & storePath) {
-            if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+        [&](const Path & storePathS) {
+            auto storePath = srcStore->parseStorePath(storePathS);
+
+            auto info = srcStore->queryPathInfo(storePath);
+            auto storePathForDst = storePath.clone();
+            if (hasPrefix(info->ca, "fixed:"))
+            {
+                FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
+                Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
+                storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+            }
+
+            if (dstStore->isValidPath(storePathForDst)) {
                 nrDone++;
                 showProgress();
                 return PathSet();
             }
 
-            auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
             bytesExpected += info->narSize;
             act.setExpected(actCopyPath, bytesExpected);
 
@@ -667,9 +676,18 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
         [&](const Path & storePathS) {
             checkInterrupt();
 
-            auto storePath = dstStore->parseStorePath(storePathS);
+            auto storePath = srcStore->parseStorePath(storePathS);
+            auto info = srcStore->queryPathInfo(storePath);
 
-            if (!dstStore->isValidPath(storePath)) {
+            auto storePathForDst = storePath.clone();
+            if (hasPrefix(info->ca, "fixed:"))
+            {
+                FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
+                Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
+                storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+            }
+
+            if (!dstStore->isValidPath(storePathForDst)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
                 try {

From 1c148a80fe05580f94e624134b923799f6cbc7b3 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 14:39:44 -0500
Subject: [PATCH 07/24] Replace --hashed-mirrors with substituters test

---
 tests/check.sh    | 14 +++++++-------
 tests/fetchurl.sh | 21 +++++++++------------
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/tests/check.sh b/tests/check.sh
index 5f25d04cb..5f4997e28 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
 [ "$status" = "1" ]
 grep 'differs from previous round' $TEST_ROOT/log
 
-path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
+path=$(nix-build check.nix -A fetchurl --no-out-link)
 
 chmod +w $path
 echo foo > $path
 chmod -w $path
 
-nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --check
 # Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
 [[ $(cat $path) = foo ]]
 
-nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --repair
 [[ $(cat $path) != foo ]]
 
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link || status=$?
 [ "$status" = "102" ]
 
 echo -n > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+nix-build check.nix -A hashmismatch --no-out-link
 echo 'Hello World' > ./dummy
 
-nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
 [ "$status" = "102" ]
 
 # Multiple failures with --keep-going
 nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
 [ "$status" = "110" ]
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 2535651b0..13447d774 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
 # Test fetching a flat file.
 hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -14,7 +14,7 @@ clearStore
 
 hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -25,26 +25,23 @@ hash=$(nix hash-file ./fetchurl.sh)
 
 [[ $hash =~ ^sha256- ]]
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
-# Test the hashed mirror feature.
+# Test that we can substitute from a different store dir.
 clearStore
 
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+other_store=file://$TEST_ROOT/other_store
 
-mirror=$TEST_ROOT/hashed-mirror
-rm -rf $mirror
-mkdir -p $mirror/sha512
-ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
+storePath=$(nix add-to-store --store $other_store ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
+outPath=$(nix-build -vvvvvv '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
 nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
-          --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+          --argstr name bla --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive

From 85d01e1f658d93a0ea8d0396b1e79d03c2ff1da2 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 14:53:30 -0500
Subject: [PATCH 08/24] =?UTF-8?q?Don=E2=80=99t=20use=20makeStorePath?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/libstore/store-api.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1b340bf6c..afa557e16 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -583,7 +583,9 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     // recompute store path on the chance dstStore does it differently
     if (hasPrefix(info->ca, "fixed:") && info->references.empty()) {
         auto info2 = make_ref<ValidPathInfo>(*info);
-        info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name());
+        FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
+        Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
+        info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
         info = info2;
     }
 

From 5b386b05f58b1ed996afe47791d85e6d20da91a2 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 14:57:45 -0500
Subject: [PATCH 09/24] Recompute storePath based on isContentAddressed

---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index afa557e16..c8dc4744f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -581,7 +581,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     uint64_t total = 0;
 
     // recompute store path on the chance dstStore does it differently
-    if (hasPrefix(info->ca, "fixed:") && info->references.empty()) {
+    if (info->isContentAddressed(*srcStore)) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
         Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));

From e3cb536f19920018b522f76ac2661d68627b7c9e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 15:25:29 -0500
Subject: [PATCH 10/24] Fix add-to-store --flat to put in correct hash

---
 src/nix/add-to-store.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index a5a77f17c..26edcf95a 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -53,14 +53,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         auto narHash = hashString(htSHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, narHash, *namePart));
-        info.narHash = narHash;
-        info.narSize = sink.s->size();
-
         Hash hash;
         switch (ingestionMethod) {
         case FileIngestionMethod::Recursive: {
-            hash = info.narHash;
+            hash = narHash;
             break;
         }
         case FileIngestionMethod::Flat: {
@@ -70,6 +66,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             break;
         }
         }
+
+        ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
+        info.narHash = narHash;
+        info.narSize = sink.s->size();
         info.ca = makeFixedOutputCA(ingestionMethod, hash);
 
         if (!dryRun) {

From 88120442d23a7e00833e8b0d523a6aa8072287b3 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 15:32:52 -0500
Subject: [PATCH 11/24] Debug when storePath changes

these rewrites should be transparent, but they are important to know
about when debugging
---
 src/libstore/store-api.cc | 4 ++++
 tests/fetchurl.sh         | 7 ++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c8dc4744f..7cb48b293 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -661,6 +661,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
                 FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
                 Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
                 storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+                if (storePathForDst != storePath)
+                    debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
 
             if (dstStore->isValidPath(storePathForDst)) {
@@ -687,6 +689,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
                 FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
                 Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
                 storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+                if (storePathForDst != storePath)
+                    debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
 
             if (!dstStore->isValidPath(storePathForDst)) {
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 13447d774..510f98843 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -35,12 +35,13 @@ clearStore
 other_store=file://$TEST_ROOT/other_store
 
 hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
-storePath=$(nix add-to-store --store $other_store ./fetchurl.sh)
 
-outPath=$(nix-build -vvvvvv '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
+
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
-nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
+nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
           --argstr name bla --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.

From 006f1252d2fdb36c7ee66bf79f0df2ab8cd16816 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 15:33:24 -0500
Subject: [PATCH 12/24] Fix SRI test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We can’t use custom name here because different names will have
different store paths. This is a limitation of the Store API’s
reliance on store paths.

We might be able to get around the above in the future by using a
dummy name for certain fixed output paths.
---
 tests/fetchurl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 510f98843..f4b86d251 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -42,7 +42,7 @@ outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchu
 
 # Test hashed mirrors with an SRI hash.
 nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
-          --argstr name bla --no-out-link --substituters $other_store
+          --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive

From b2cb288cdd462cbca35ffd3afda9d7ec2d595209 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 16:36:35 -0500
Subject: [PATCH 13/24] Add makeFixedOutputPathFromCA function

This puts what we are already doing into a shared method. It just
needs a path name and a ca and produces a store path.
---
 src/libstore/local-store.cc | 11 +++--------
 src/libstore/store-api.cc   | 31 ++++++++++++++++++-------------
 src/libstore/store-api.hh   |  6 +++++-
 3 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index bf005a3b3..9158a52e0 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -862,15 +862,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
         for (auto & path : paths) {
             auto subPath(path.clone());
 
-            auto ca_ = pathsCA.find(printStorePath(path));
+            auto ca = pathsCA.find(printStorePath(path));
             // recompute store path so that we can use a different store root
-            if (ca_ != pathsCA.end()) {
-                auto ca(ca_->second);
-                if (!hasPrefix(ca, "fixed:"))
-                    continue;
-                FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 };
-                Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
-                subPath = makeFixedOutputPath(ingestionMethod, hash, path.name());
+            if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) {
+                subPath = makeFixedOutputPathFromCA(path.name(), ca->second);
                 if (subPath != path)
                     debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri());
             } else if (sub->storeDir != storeDir) continue;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7cb48b293..bc9a6c4f5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -191,6 +191,19 @@ StorePath Store::makeFixedOutputPath(
     }
 }
 
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, std::string ca,
+    const StorePathSet & references, bool hasSelfReference) const
+{
+    if (hasPrefix(ca, "fixed:")) {
+        FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 };
+        Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
+        return makeFixedOutputPath(ingestionMethod, hash, name, references, hasSelfReference);
+    } else if (hasPrefix(ca, "text:")) {
+        Hash hash(std::string(ca, 5));
+        return makeTextPath(name, hash, references);
+    } else
+        throw Error("'%s' is not a valid ca", ca);
+}
 
 StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
     const StorePathSet & references) const
@@ -583,9 +596,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     // recompute store path on the chance dstStore does it differently
     if (info->isContentAddressed(*srcStore)) {
         auto info2 = make_ref<ValidPathInfo>(*info);
-        FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
-        Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
-        info2->path = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+        info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca);
         info = info2;
     }
 
@@ -656,11 +667,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
 
             auto info = srcStore->queryPathInfo(storePath);
             auto storePathForDst = storePath.clone();
-            if (hasPrefix(info->ca, "fixed:"))
-            {
-                FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
-                Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
-                storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+            if (info->isContentAddressed(*srcStore)) {
+                storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (storePathForDst != storePath)
                     debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
@@ -684,11 +692,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             auto info = srcStore->queryPathInfo(storePath);
 
             auto storePathForDst = storePath.clone();
-            if (hasPrefix(info->ca, "fixed:"))
-            {
-                FileIngestionMethod ingestionMethod { info->ca.compare(6, 2, "r:") == 0 };
-                Hash hash(std::string(info->ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
-                storePathForDst = dstStore->makeFixedOutputPath(ingestionMethod, hash, storePath.name());
+            if (info->isContentAddressed(*srcStore)) {
+                storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (storePathForDst != storePath)
                     debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6f961329c..9412785b4 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -352,7 +352,11 @@ public:
         bool hasSelfReference = false) const;
 
     StorePath makeTextPath(std::string_view name, const Hash & hash,
-        const StorePathSet & references) const;
+        const StorePathSet & references = {}) const;
+
+    StorePath makeFixedOutputPathFromCA(std::string_view name, std::string ca,
+        const StorePathSet & references = {},
+        bool hasSelfReference = false) const;
 
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store

From c214cda9401cf50d0419038746428260b0dfdd63 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 13 Jun 2020 00:07:42 -0500
Subject: [PATCH 14/24] Correctly substitute from different storeDir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Originally, the test was only checking for different “real” storeDir.
That’s an easy case to handle, but the much harder one is if different
virtual store dirs are used. To do this, we need the SubstitutionGoal
to know about the ca, so it can recalculate the path to copy it over.
An important note here is that the store path passed to copyStorePath
needs to be one for srcStore - so that queryPathInfo works properly.

This also adds an error message when the store path from queryPathInfo
is different from the one we requested.
---
 src/libstore/build.cc     | 44 ++++++++++++++++++++++++++++-----------
 src/libstore/misc.cc      | 12 +++++------
 src/libstore/store-api.hh |  2 ++
 tests/fetchurl.sh         |  4 ++--
 4 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f5c132a83..f3f6f01cc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -305,7 +305,7 @@ public:
     GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath,
         const BasicDerivation & drv, BuildMode buildMode = bmNormal);
-    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
+    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);
@@ -1195,7 +1195,7 @@ void DerivationGoal::haveDerivation()
        them. */
     if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
         for (auto & i : invalidOutputs)
-            addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+            addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
@@ -4268,8 +4268,11 @@ private:
     typedef void (SubstitutionGoal::*GoalState)();
     GoalState state;
 
+    /* Content address for recomputing store path */
+    std::optional<std::string> ca;
+
 public:
-    SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair);
+    SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt);
     ~SubstitutionGoal();
 
     void timedOut() override { abort(); };
@@ -4304,10 +4307,11 @@ public:
 };
 
 
-SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair, std::optional<std::string> ca)
     : Goal(worker)
     , storePath(std::move(storePath))
     , repair(repair)
+    , ca(ca)
 {
     state = &SubstitutionGoal::init;
     name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@@ -4382,14 +4386,13 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    if (sub->storeDir != worker.store.storeDir) {
-        tryNext();
-        return;
-    }
+    auto subPath = storePath.clone();
+    if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:")))
+        subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
 
     try {
         // FIXME: make async
-        info = sub->queryPathInfo(storePath);
+        info = sub->queryPathInfo(subPath);
     } catch (InvalidPath &) {
         tryNext();
         return;
@@ -4408,6 +4411,19 @@ void SubstitutionGoal::tryNext()
         throw;
     }
 
+    if (info->path != storePath) {
+        if (info->isContentAddressed(*sub)) {
+            auto info2 = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(info));
+            info2->path = storePath.clone();
+            info = info2;
+        } else {
+            printError("asked '%s' for '%s' but got '%s'",
+                sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+            tryNext();
+            return;
+        }
+    }
+
     /* Update the total expected download size. */
     auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
@@ -4493,8 +4509,12 @@ void SubstitutionGoal::tryToRun()
             Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
 
+            auto subPath = storePath.clone();
+            if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:")))
+                subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+                subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {
@@ -4628,11 +4648,11 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drv
 }
 
 
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<std::string> ca)
 {
     GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME
     if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair);
+        goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair, ca);
         substitutionGoals.insert_or_assign(path.clone(), goal);
         wakeUp(goal);
     }
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 1b0a055d3..109e9e473 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,12 +108,12 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
-static std::optional<std::string> getDerivationCA(ref<Derivation> drv)
+std::optional<std::string> getDerivationCA(const BasicDerivation & drv)
 {
-    auto outputHashMode = drv->env.find("outputHashMode");
-    auto outputHashAlgo = drv->env.find("outputHashAlgo");
-    auto outputHash = drv->env.find("outputHash");
-    if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) {
+    auto outputHashMode = drv.env.find("outputHashMode");
+    auto outputHashAlgo = drv.env.find("outputHashAlgo");
+    auto outputHash = drv.env.find("outputHash");
+    if (outputHashMode != drv.env.end() && outputHashAlgo != drv.env.end() && outputHash != drv.env.end()) {
         auto ht = parseHashType(outputHashAlgo->second);
         auto h = Hash(outputHash->second, ht);
         FileIngestionMethod ingestionMethod;
@@ -182,7 +182,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         paths.insert(outPath.clone());
 
         std::map<std::string, std::string> pathsCA = {};
-        if (auto ca = getDerivationCA(drv))
+        if (auto ca = getDerivationCA(*drv))
             pathsCA.insert({outPathS, *ca});
         querySubstitutablePathInfos(paths, infos, pathsCA);
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9412785b4..9918a36c6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -855,4 +855,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
 /* Split URI into protocol+hierarchy part and its parameter set. */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
 
+std::optional<std::string> getDerivationCA(const BasicDerivation & drv);
+
 }
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index f4b86d251..aaa38f2ed 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -32,13 +32,13 @@ cmp $outPath fetchurl.sh
 # Test that we can substitute from a different store dir.
 clearStore
 
-other_store=file://$TEST_ROOT/other_store
+other_store=file://$TEST_ROOT/other_store?store=/fnord/store
 
 hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
 
 storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+outPath=$(nix-build -vvvvv '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
 nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \

From 9ac0d98a59ed042f698685234109d4b69bccf29c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 13 Jun 2020 00:14:30 -0500
Subject: [PATCH 15/24] Remove -vvvvv from tests/fetchurl.sh nix-build call

---
 tests/fetchurl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index aaa38f2ed..0f2044342 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -38,7 +38,7 @@ hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
 
 storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
 
-outPath=$(nix-build -vvvvv '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
 nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \

From 6438ba1e990ddf76602a70d5c1143b73ed31855c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 17 Jun 2020 12:31:01 -0400
Subject: [PATCH 16/24] Update strings from review comment

---
 src/libstore/store-api.cc | 4 ++--
 src/nix/add-to-store.cc   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index bc9a6c4f5..2593b245c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -670,7 +670,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             if (info->isContentAddressed(*srcStore)) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (storePathForDst != storePath)
-                    debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+                    debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
 
             if (dstStore->isValidPath(storePathForDst)) {
@@ -695,7 +695,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             if (info->isContentAddressed(*srcStore)) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (storePathForDst != storePath)
-                    debug("rewriting path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+                    debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
 
             if (!dstStore->isValidPath(storePathForDst)) {
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 26edcf95a..7abb82556 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -26,7 +26,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         addFlag({
             .longName = "flat",
             .shortName = 0,
-            .description = "use flat file ingestion",
+            .description = "add flat file to the Nix store",
             .handler = {&ingestionMethod, FileIngestionMethod::Flat},
         });
     }

From 8974755d1958824e732640f8131b0ed22ebd703b Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 17 Jun 2020 14:04:46 -0400
Subject: [PATCH 17/24] Add assert for replaced storePath

---
 src/libstore/build.cc       | 10 ++++++++--
 src/libstore/local-store.cc |  2 ++
 src/libstore/store-api.cc   |  6 ++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9753db7b1..bc67ae31e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4409,8 +4409,11 @@ void SubstitutionGoal::tryNext()
     subs.pop_front();
 
     auto subPath = storePath;
-    if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:")))
+    if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) {
         subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+        if (sub->storeDir == worker.store.storeDir)
+            assert(subPath == storePath);
+    }
 
     try {
         // FIXME: make async
@@ -4535,8 +4538,11 @@ void SubstitutionGoal::tryToRun()
             PushActivity pact(act.id);
 
             auto subPath = storePath;
-            if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:")))
+            if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) {
                 subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+                if (sub->storeDir == worker.store.storeDir)
+                    assert(subPath == storePath);
+            }
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
                 subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a80026258..d08c7dc6a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -853,6 +853,8 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
             // recompute store path so that we can use a different store root
             if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) {
                 subPath = makeFixedOutputPathFromCA(path.name(), ca->second);
+                if (sub->storeDir == storeDir)
+                    assert(subPath == path);
                 if (subPath != path)
                     debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri());
             } else if (sub->storeDir != storeDir) continue;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a28895086..80a69717e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -598,6 +598,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     if (info->isContentAddressed(*srcStore)) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca);
+        if (dstStore->storeDir == srcStore->storeDir)
+            assert(info->path == info2->path);
         info = info2;
     }
 
@@ -670,6 +672,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             auto storePathForDst = storePath;
             if (info->isContentAddressed(*srcStore)) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
+                if (dstStore->storeDir == srcStore->storeDir)
+                    assert(storePathForDst == storePath);
                 if (storePathForDst != storePath)
                     debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
@@ -695,6 +699,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             auto storePathForDst = storePath;
             if (info->isContentAddressed(*srcStore)) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
+                if (dstStore->storeDir == srcStore->storeDir)
+                    assert(storePathForDst == storePath);
                 if (storePathForDst != storePath)
                     debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }

From be50de1142b0600afebde32130e8561e28b63b41 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 17 Jun 2020 14:14:22 -0400
Subject: [PATCH 18/24] Make sure references are empty for store path replacing

also copy info2 instead of casting
---
 src/libstore/build.cc       | 4 ++--
 src/libstore/local-store.cc | 2 +-
 src/libstore/store-api.cc   | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index bc67ae31e..63cb70ef2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4437,8 +4437,8 @@ void SubstitutionGoal::tryNext()
     }
 
     if (info->path != storePath) {
-        if (info->isContentAddressed(*sub)) {
-            auto info2 = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(info));
+        if (info->isContentAddressed(*sub) && info->references.empty()) {
+            auto info2 = std::make_shared<ValidPathInfo>(*info);
             info2->path = storePath;
             info = info2;
         } else {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d08c7dc6a..3230312c9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -863,7 +863,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
             try {
                 auto info = sub->queryPathInfo(subPath);
 
-                if (sub->storeDir != storeDir && !info->isContentAddressed(*sub))
+                if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
                     continue;
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 80a69717e..c8097da52 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -595,7 +595,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     uint64_t total = 0;
 
     // recompute store path on the chance dstStore does it differently
-    if (info->isContentAddressed(*srcStore)) {
+    if (info->isContentAddressed(*srcStore) && info->references.empty()) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca);
         if (dstStore->storeDir == srcStore->storeDir)
@@ -670,7 +670,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
 
             auto info = srcStore->queryPathInfo(storePath);
             auto storePathForDst = storePath;
-            if (info->isContentAddressed(*srcStore)) {
+            if (info->isContentAddressed(*srcStore) && info->references.empty()) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (dstStore->storeDir == srcStore->storeDir)
                     assert(storePathForDst == storePath);
@@ -697,7 +697,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             auto info = srcStore->queryPathInfo(storePath);
 
             auto storePathForDst = storePath;
-            if (info->isContentAddressed(*srcStore)) {
+            if (info->isContentAddressed(*srcStore) && info->references.empty()) {
                 storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
                 if (dstStore->storeDir == srcStore->storeDir)
                     assert(storePathForDst == storePath);

From 5e631e3304aaf00b912c12025debfd35942a0ca9 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 17 Jun 2020 15:03:05 -0400
Subject: [PATCH 19/24] Add StorePathCAMap for querySubstitutablePathInfos
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I’m not 100% sure this is wanted since it kind of makes everything
have to know about ca even if they don’t really want to. But it also
make things easier in dealing with looking up ca.
---
 src/libstore/daemon.cc       |  7 +++++--
 src/libstore/local-store.cc  | 18 ++++++++----------
 src/libstore/local-store.hh  |  5 ++---
 src/libstore/misc.cc         |  7 ++-----
 src/libstore/path.hh         |  2 ++
 src/libstore/remote-store.cc | 14 ++++++++------
 src/libstore/remote-store.hh |  4 ++--
 src/libstore/store-api.hh    |  4 ++--
 8 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index e370e278c..f7dc16948 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos({path}, infos);
+        store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
         logger->stopWork();
         auto i = infos.find(path);
         if (i == infos.end())
@@ -612,7 +612,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto paths = readStorePaths<StorePathSet>(*store, from);
         logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos(paths, infos);
+        StorePathCAMap pathsMap = {};
+        for (auto & path : paths)
+            pathsMap.emplace(path, std::nullopt);
+        store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
         to << infos.size();
         for (auto & i : infos) {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3230312c9..3901d532a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -841,22 +841,20 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 }
 
 
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
         for (auto & path : paths) {
-            auto subPath(path);
+            auto subPath(path.first);
 
-            auto ca = pathsCA.find(printStorePath(path));
             // recompute store path so that we can use a different store root
-            if (ca != pathsCA.end() && (hasPrefix(ca->second, "fixed:") || hasPrefix(ca->second, "text:"))) {
-                subPath = makeFixedOutputPathFromCA(path.name(), ca->second);
+            if (path.second && (hasPrefix(*path.second, "fixed:") || hasPrefix(*path.second, "text:"))) {
+                subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
                 if (sub->storeDir == storeDir)
-                    assert(subPath == path);
-                if (subPath != path)
-                    debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path), sub->printStorePath(subPath), sub->getUri());
+                    assert(subPath == path.first);
+                if (subPath != path.first)
+                    debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
             } else if (sub->storeDir != storeDir) continue;
 
             debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
@@ -868,7 +866,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
-                infos.insert_or_assign(path, SubstitutablePathInfo{
+                infos.insert_or_assign(path.first, SubstitutablePathInfo{
                     info->deriver,
                     info->references,
                     narInfo ? narInfo->fileSize : 0,
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 183d27c80..ab9fbfbe4 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -139,9 +139,8 @@ public:
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos,
-        std::map<std::string, std::string> pathsCA = {}) override;
+    void querySubstitutablePathInfos(const StorePathCAMap & paths,
+        SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 6902dfa0c..dc991e227 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -178,10 +178,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         auto outPath = parseStorePath(outPathS);
 
         SubstitutablePathInfos infos;
-        std::map<std::string, std::string> pathsCA = {};
-        if (auto ca = getDerivationCA(*drv))
-            pathsCA.insert({outPathS, *ca});
-        querySubstitutablePathInfos({outPath}, infos, pathsCA);
+        querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;
@@ -238,7 +235,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (isValidPath(path.path)) return;
 
             SubstitutablePathInfos infos;
-            querySubstitutablePathInfos({path.path}, infos);
+            querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
 
             if (infos.empty()) {
                 auto state(state_.lock());
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index aaebd3ec3..91962d114 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -62,6 +62,8 @@ public:
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
 
+typedef std::map<StorePath, std::optional<std::string>> StorePathCAMap;
+
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 396629a32..4538ccd4f 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -308,18 +308,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 }
 
 
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
 {
-    if (paths.empty()) return;
+    if (pathsMap.empty()) return;
 
     auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
 
-        for (auto & i : paths) {
+        for (auto & i : pathsMap) {
             SubstitutablePathInfo info;
-            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
             conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
@@ -329,12 +328,15 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
             info.references = readStorePaths<StorePathSet>(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
-            infos.insert_or_assign(i, std::move(info));
+            infos.insert_or_assign(i.first, std::move(info));
         }
 
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos;
+        StorePathSet paths;
+        for (auto & path : pathsMap)
+            paths.insert(path.first);
         writeStorePaths(*this, conn->to, paths);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 498a5c6a0..07ed2b51a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -55,8 +55,8 @@ public:
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) override;
+    void querySubstitutablePathInfos(const StorePathCAMap & paths,
+        SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 65954574f..12a6b6939 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -445,8 +445,8 @@ public:
     /* Query substitute info (i.e. references, derivers and download
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
-    virtual void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) { return; };
+    virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
+        SubstitutablePathInfos & infos) { return; };
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,

From 0c9c1b8826395a9023606cd7749aa16ef6771941 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 19 Jun 2020 14:48:57 -0400
Subject: [PATCH 20/24] Return map of StorePaths in copyPaths

This allows the caller to know what values were actually added to the
store.
---
 src/libstore/store-api.cc | 12 ++++++++++--
 src/libstore/store-api.hh | 17 ++++++++++-------
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 661dea141..0458fd9a6 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -638,7 +638,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
     auto valid = dstStore->queryValidPaths(storePaths, substitute);
@@ -647,7 +647,11 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
 
-    if (missing.empty()) return;
+    std::map<StorePath, StorePath> pathsMap;
+    for (auto & path : storePaths)
+        pathsMap.insert_or_assign(path, path);
+
+    if (missing.empty()) return pathsMap;
 
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
@@ -677,6 +681,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
                 if (storePathForDst != storePath)
                     debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
+            pathsMap.insert_or_assign(storePath, storePathForDst);
 
             if (dstStore->isValidPath(storePathForDst)) {
                 nrDone++;
@@ -704,6 +709,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
                 if (storePathForDst != storePath)
                     debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
             }
+            pathsMap.insert_or_assign(storePath, storePathForDst);
 
             if (!dstStore->isValidPath(storePathForDst)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
@@ -723,6 +729,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
             nrDone++;
             showProgress();
         });
+
+    return pathsMap;
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index fd3652588..2b12879e3 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -444,8 +444,9 @@ public:
     virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
 
     /* Query substitute info (i.e. references, derivers and download
-       sizes) of a set of paths.  If a path does not have substitute
-       info, it's omitted from the resulting ‘infos’ map. */
+       sizes) of a map of paths to their optional ca values. If a path
+       does not have substitute info, it's omitted from the resulting
+       ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) { return; };
 
@@ -743,11 +744,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
 
 /* Copy store paths from one store to another. The paths may be copied
-   in parallel. They are copied in a topologically sorted order
-   (i.e. if A is a reference of B, then A is copied before B), but
-   the set of store paths is not automatically closed; use
-   copyClosure() for that. */
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+   in parallel. They are copied in a topologically sorted order (i.e.
+   if A is a reference of B, then A is copied before B), but the set
+   of store paths is not automatically closed; use copyClosure() for
+   that. Returns a map of what each path was copied to the dstStore
+   as. */
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+    const StorePathSet & storePaths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);

From e8e1f5282ff213be010cbe45772b9e725c01f285 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 19 Jun 2020 15:19:09 -0400
Subject: [PATCH 21/24] Replace error message in getDerivationCA

---
 src/libstore/misc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 128c3c987..a01e81b78 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -126,7 +126,7 @@ std::optional<std::string> getDerivationCA(const BasicDerivation & drv)
         else if (outputHashMode->second == "flat")
             ingestionMethod = FileIngestionMethod::Flat;
         else
-            throw Error("unknown ingestion method: '%s'", outputHashMode->second);
+            throw Error("unknown outputHashMode: '%s'", outputHashMode->second);
         return makeFixedOutputCA(ingestionMethod, h);
     }
 

From f2a6cee334255ced72a70f527cf3c283b4586e42 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 19 Jun 2020 18:06:19 -0400
Subject: [PATCH 22/24] Update worker protocol to support sending storepath
 maps

We need to also send the ca to daemon in addition to the path.
---
 src/libstore/daemon.cc          | 11 +++++++----
 src/libstore/remote-store.cc    | 32 ++++++++++++++++++++++++++++----
 src/libstore/worker-protocol.hh |  6 +++++-
 3 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index f7dc16948..c613ab47d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -609,12 +609,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePathInfos: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
-        logger->startWork();
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
-        for (auto & path : paths)
-            pathsMap.emplace(path, std::nullopt);
+        if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+            auto paths = readStorePaths<StorePathSet>(*store, from);
+            for (auto & path : paths)
+                pathsMap.emplace(path, std::nullopt);
+        } else
+            pathsMap = readStorePathCAMap(*store, from);
+        logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
         to << infos.size();
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 4538ccd4f..f6158685c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -38,6 +38,27 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
         out << store.printStorePath(i);
 }
 
+std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from)
+{
+    StorePathCAMap paths;
+    auto count = readNum<size_t>(from);
+    while (count--) {
+        auto path = readString(from);
+        auto ca = readString(from);
+        paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt);
+    }
+    return paths;
+}
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
+{
+    out << paths.size();
+    for (auto & i : paths) {
+        out << store.printStorePath(i.first);
+        out << (i.second ? *i.second : "");
+    }
+}
+
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -334,10 +355,13 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos;
-        StorePathSet paths;
-        for (auto & path : pathsMap)
-            paths.insert(path.first);
-        writeStorePaths(*this, conn->to, paths);
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+            StorePathSet paths;
+            for (auto & path : pathsMap)
+                paths.insert(path.first);
+            writeStorePaths(*this, conn->to, paths);
+        } else
+            writeStorePathCAMap(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index ac42457fc..d2944674f 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x115
+#define PROTOCOL_VERSION 0x116
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -69,5 +69,9 @@ template<class T> T readStorePaths(const Store & store, Source & from);
 
 void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
 
+std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from);
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
+
 
 }

From 1f9cb06db23f357f06404969820be1d3d4118a18 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 2 Jul 2020 10:59:24 -0400
Subject: [PATCH 23/24] Try next when no ca exists and have different store
 dirs

---
 src/libstore/build.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 95280ef1e..20d3c7387 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4421,6 +4421,9 @@ void SubstitutionGoal::tryNext()
         subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
         if (sub->storeDir == worker.store.storeDir)
             assert(subPath == storePath);
+    } else if (sub->storeDir != worker.store.storeDir) {
+        tryNext();
+        return;
     }
 
     try {

From d2e8b9ff0e8048d7b3914811b2d4bd535feac49d Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 2 Jul 2020 11:12:05 -0400
Subject: [PATCH 24/24] Store subPath in SubstitutionGoal

---
 src/libstore/build.cc | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 20d3c7387..12e4b6c3b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4270,6 +4270,10 @@ private:
     /* The store path that should be realised through a substitute. */
     StorePath storePath;
 
+    /* The path the substituter refers to the path as. This will be
+     * different when the stores have different names. */
+    std::optional<StorePath> subPath;
+
     /* The remaining substituters. */
     std::list<ref<Store>> subs;
 
@@ -4416,7 +4420,6 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    auto subPath = storePath;
     if (ca) {
         subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
         if (sub->storeDir == worker.store.storeDir)
@@ -4428,7 +4431,7 @@ void SubstitutionGoal::tryNext()
 
     try {
         // FIXME: make async
-        info = sub->queryPathInfo(subPath);
+        info = sub->queryPathInfo(subPath ? *subPath : storePath);
     } catch (InvalidPath &) {
         tryNext();
         return;
@@ -4548,15 +4551,8 @@ void SubstitutionGoal::tryToRun()
             Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
 
-            auto subPath = storePath;
-            if (ca) {
-                subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
-                if (sub->storeDir == worker.store.storeDir)
-                    assert(subPath == storePath);
-            }
-
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+                subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {