Merge pull request #6302 from edolstra/fetch-closure

Add builtins.fetchClosure

doc/manual/src/release-notes/rl-next.md

@@ -1,3 +1,13 @@
 # Release X.Y (202?-??-??)
 
 * Various nix commands can now read expressions from stdin with `--file -`.
+
+* `nix store make-content-addressable` has been renamed to `nix store
+  make-content-addressed`.
+
+* New builtin function `builtins.fetchClosure` that copies a closure
+  from a binary cache at evaluation time and rewrites it to
+  content-addressed form (if it isn't already). Like
+  `builtins.storePath`, this allows importing pre-built store paths;
+  the difference is that it doesn't require the user to configure
+  binary caches and trusted public keys.

src/libexpr/primops/fetchClosure.cc

@@ -0,0 +1,152 @@
+#include "primops.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "url.hh"
+
+namespace nix {
+
+static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature(Xp::FetchClosure);
+
+    state.forceAttrs(*args[0], pos);
+
+    std::optional<std::string> fromStoreUrl;
+    std::optional<StorePath> fromPath;
+    bool toCA = false;
+    std::optional<StorePath> toPath;
+
+    for (auto & attr : *args[0]->attrs) {
+        if (attr.name == "fromPath") {
+            PathSet context;
+            fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+        }
+
+        else if (attr.name == "toPath") {
+            state.forceValue(*attr.value, *attr.pos);
+            toCA = true;
+            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+                PathSet context;
+                toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+            }
+        }
+
+        else if (attr.name == "fromStore")
+            fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
+
+        else
+            throw Error({
+                .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
+                .errPos = pos
+            });
+    }
+
+    if (!fromPath)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
+            .errPos = pos
+        });
+
+    if (!fromStoreUrl)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
+            .errPos = pos
+        });
+
+    auto parsedURL = parseURL(*fromStoreUrl);
+
+    if (parsedURL.scheme != "http" &&
+        parsedURL.scheme != "https" &&
+        !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
+        throw Error({
+            .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
+            .errPos = pos
+        });
+
+    auto fromStore = openStore(parsedURL.to_string());
+
+    if (toCA) {
+        if (!toPath || !state.store->isValidPath(*toPath)) {
+            auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+            auto i = remappings.find(*fromPath);
+            assert(i != remappings.end());
+            if (toPath && *toPath != i->second)
+                throw Error({
+                    .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second),
+                        state.store->printStorePath(*toPath)),
+                    .errPos = pos
+                });
+            if (!toPath)
+                throw Error({
+                    .msg = hintfmt(
+                        "rewriting '%s' to content-addressed form yielded '%s'; "
+                        "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second)),
+                    .errPos = pos
+                });
+        }
+    } else {
+        copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+        toPath = fromPath;
+    }
+
+    /* In pure mode, require a CA path. */
+    if (evalSettings.pureEval) {
+        auto info = state.store->queryPathInfo(*toPath);
+        if (!info->isContentAddressed(*state.store))
+            throw Error({
+                .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
+                    state.store->printStorePath(*toPath)),
+                .errPos = pos
+            });
+    }
+
+    auto toPathS = state.store->printStorePath(*toPath);
+    v.mkString(toPathS, {toPathS});
+}
+
+static RegisterPrimOp primop_fetchClosure({
+    .name = "__fetchClosure",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a Nix store closure from a binary cache, rewriting it into
+      content-addressed form. For example,
+
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+        toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+      }
+      ```
+
+      fetches `/nix/store/r2jd...` from the specified binary cache,
+      and rewrites it into the content-addressed store path
+      `/nix/store/ldbh...`.
+
+      If `fromPath` is already content-addressed, or if you are
+      allowing impure evaluation (`--impure`), then `toPath` may be
+      omitted.
+
+      To find out the correct value for `toPath` given a `fromPath`,
+      you can use `nix store make-content-addressed`:
+
+      ```console
+      # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+      rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+      ```
+
+      This function is similar to `builtins.storePath` in that it
+      allows you to use a previously built store path in a Nix
+      expression. However, it is more reproducible because it requires
+      specifying a binary cache from which the path can be fetched.
+      Also, requiring a content-addressed final store path avoids the
+      need for users to configure binary cache public keys.
+    )",
+    .fun = prim_fetchClosure,
+});
+
+}

src/libexpr/primops/fetchTree.cc

@@ -145,7 +145,7 @@ static void fetchTree(
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 throw Error({
-                    .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                    .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
                     .errPos = pos
                 });
 
@@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({
     .fun = prim_fetchTarball,
 });
 
-static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }

src/libstore/make-content-addressed.cc

@@ -0,0 +1,80 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths)
+{
+    StorePathSet closure;
+    srcStore.computeFSClosure(storePaths, closure);
+
+    auto paths = srcStore.topoSortPaths(closure);
+
+    std::reverse(paths.begin(), paths.end());
+
+    std::map<StorePath, StorePath> remappings;
+
+    for (auto & path : paths) {
+        auto pathS = srcStore.printStorePath(path);
+        auto oldInfo = srcStore.queryPathInfo(path);
+        std::string oldHashPart(path.hashPart());
+
+        StringSink sink;
+        srcStore.narFromPath(path, sink);
+
+        StringMap rewrites;
+
+        StorePathSet references;
+        bool hasSelfReference = false;
+        for (auto & ref : oldInfo->references) {
+            if (ref == path)
+                hasSelfReference = true;
+            else {
+                auto i = remappings.find(ref);
+                auto replacement = i != remappings.end() ? i->second : ref;
+                // FIXME: warn about unremapped paths?
+                if (replacement != ref)
+                    rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+                references.insert(std::move(replacement));
+            }
+        }
+
+        sink.s = rewriteStrings(sink.s, rewrites);
+
+        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        hashModuloSink(sink.s);
+
+        auto narModuloHash = hashModuloSink.finish().first;
+
+        auto dstPath = dstStore.makeFixedOutputPath(
+            FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
+
+        printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
+
+        StringSink sink2;
+        RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
+        rsink2(sink.s);
+        rsink2.flush();
+
+        ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
+        info.references = std::move(references);
+        if (hasSelfReference) info.references.insert(info.path);
+        info.narSize = sink.s.size();
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Recursive,
+            .hash = narModuloHash,
+        };
+
+        StringSource source(sink2.s);
+        dstStore.addToStore(info, source);
+
+        remappings.insert_or_assign(std::move(path), std::move(info.path));
+    }
+
+    return remappings;
+}
+
+}

src/libstore/make-content-addressed.hh

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths);
+
+}

src/libutil/experimental-features.cc

@@ -11,6 +11,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::NixCommand, "nix-command" },
     { Xp::RecursiveNix, "recursive-nix" },
     { Xp::NoUrlLiterals, "no-url-literals" },
+    { Xp::FetchClosure, "fetch-closure" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

src/libutil/experimental-features.hh

@@ -19,7 +19,8 @@ enum struct ExperimentalFeature
     Flakes,
     NixCommand,
     RecursiveNix,
-    NoUrlLiterals
+    NoUrlLiterals,
+    FetchClosure,
 };
 
 /**

src/nix/main.cc

@@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         {"hash-path", {"hash", "path"}},
         {"ls-nar", {"nar", "ls"}},
         {"ls-store", {"store", "ls"}},
-        {"make-content-addressable", {"store", "make-content-addressable"}},
+        {"make-content-addressable", {"store", "make-content-addressed"}},
         {"optimise-store", {"store", "optimise"}},
         {"ping-store", {"store", "ping"}},
         {"sign-paths", {"store", "sign"}},

src/nix/make-content-addressable.cc

@@ -1,102 +0,0 @@
-#include "command.hh"
-#include "store-api.hh"
-#include "references.hh"
-#include "common-args.hh"
-#include "json.hh"
-
-using namespace nix;
-
-struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
-{
-    CmdMakeContentAddressable()
-    {
-        realiseMode = Realise::Outputs;
-    }
-
-    std::string description() override
-    {
-        return "rewrite a path or closure to content-addressed form";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "make-content-addressable.md"
-          ;
-    }
-
-    void run(ref<Store> store, StorePaths && storePaths) override
-    {
-        auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
-
-        std::reverse(paths.begin(), paths.end());
-
-        std::map<StorePath, StorePath> remappings;
-
-        auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
-        auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
-
-        for (auto & path : paths) {
-            auto pathS = store->printStorePath(path);
-            auto oldInfo = store->queryPathInfo(path);
-            std::string oldHashPart(path.hashPart());
-
-            StringSink sink;
-            store->narFromPath(path, sink);
-
-            StringMap rewrites;
-
-            StorePathSet references;
-            bool hasSelfReference = false;
-            for (auto & ref : oldInfo->references) {
-                if (ref == path)
-                    hasSelfReference = true;
-                else {
-                    auto i = remappings.find(ref);
-                    auto replacement = i != remappings.end() ? i->second : ref;
-                    // FIXME: warn about unremapped paths?
-                    if (replacement != ref)
-                        rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
-                    references.insert(std::move(replacement));
-                }
-            }
-
-            sink.s = rewriteStrings(sink.s, rewrites);
-
-            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-            hashModuloSink(sink.s);
-
-            auto narHash = hashModuloSink.finish().first;
-
-            ValidPathInfo info {
-                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
-                narHash,
-            };
-            info.references = std::move(references);
-            if (hasSelfReference) info.references.insert(info.path);
-            info.narSize = sink.s.size();
-            info.ca = FixedOutputHash {
-                .method = FileIngestionMethod::Recursive,
-                .hash = info.narHash,
-            };
-
-            if (!json)
-                notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
-            auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
-                rsink2(sink.s);
-                rsink2.flush();
-            });
-
-            store->addToStore(info, *source);
-
-            if (json)
-                jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
-            remappings.insert_or_assign(std::move(path), std::move(info.path));
-        }
-    }
-};
-
-static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});

src/nix/make-content-addressed.cc

@@ -0,0 +1,55 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "common-args.hh"
+#include "json.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
+{
+    CmdMakeContentAddressed()
+    {
+        realiseMode = Realise::Outputs;
+    }
+
+    std::string description() override
+    {
+        return "rewrite a path or closure to content-addressed form";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "make-content-addressed.md"
+          ;
+    }
+
+    void run(ref<Store> srcStore, StorePaths && storePaths) override
+    {
+        auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+
+        auto remappings = makeContentAddressed(*srcStore, *dstStore,
+            StorePathSet(storePaths.begin(), storePaths.end()));
+
+        if (json) {
+            JSONObject jsonRoot(std::cout);
+            JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
+            }
+        } else {
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                notice("rewrote '%s' to '%s'",
+                    srcStore->printStorePath(path),
+                    srcStore->printStorePath(i->second));
+            }
+        }
+    }
+};
+
+static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});

src/nix/make-content-addressed.md

@@ -5,7 +5,7 @@ R""(
 * Create a content-addressed representation of the closure of GNU Hello:
 
   ```console
-  # nix store make-content-addressable -r nixpkgs#hello
+  # nix store make-content-addressed nixpkgs#hello
   …
   rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
   ```
@@ -29,7 +29,7 @@ R""(
   system closure:
 
   ```console
-  # nix store make-content-addressable -r /run/current-system
+  # nix store make-content-addressed /run/current-system
   ```
 
 # Description

tests/binary-cache.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-needLocalStore "“--no-require-sigs” can’t be used with the daemon"
+needLocalStore "'--no-require-sigs' can’t be used with the daemon"
 
 # We can produce drvs directly into the binary cache
 clearStore

tests/fetchClosure.sh

@@ -0,0 +1,58 @@
+source common.sh
+
+enableFeatures "fetch-closure"
+needLocalStore "'--no-require-sigs' can’t be used with the daemon"
+
+clearStore
+clearCacheCache
+
+# Initialize binary cache.
+nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
+caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
+nix copy --to file://$cacheDir $nonCaPath
+
+# Test basic fetchClosure rewriting from non-CA to CA.
+clearStore
+
+[ ! -e $nonCaPath ]
+[ ! -e $caPath ]
+
+[[ $(nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+    toPath = $caPath;
+  }
+") = $caPath ]]
+
+[ ! -e $nonCaPath ]
+[ -e $caPath ]
+
+# In impure mode, we can use non-CA paths.
+[[ $(nix eval --raw --no-require-sigs --impure --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+  }
+") = $nonCaPath ]]
+
+[ -e $nonCaPath ]
+
+# 'toPath' set to empty string should fail but print the expected path.
+nix eval -v --json --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+    toPath = \"\";
+  }
+" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath"
+
+# If fromPath is CA, then toPath isn't needed.
+nix copy --to file://$cacheDir $caPath
+
+[[ $(nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $caPath;
+  }
+") = $caPath ]]

tests/local.mk

@@ -96,7 +96,8 @@ nix_tests = \
   describe-stores.sh \
   nix-profile.sh \
   suggestions.sh \
-  store-ping.sh
+  store-ping.sh \
+  fetchClosure.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh