diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c9753f9aa..33eaa8a2c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,13 @@ # Release X.Y (202?-??-??) * Various nix commands can now read expressions from stdin with `--file -`. + +* `nix store make-content-addressable` has been renamed to `nix store + make-content-addressed`. + +* New builtin function `builtins.fetchClosure` that copies a closure + from a binary cache at evaluation time and rewrites it to + content-addressed form (if it isn't already). Like + `builtins.storePath`, this allows importing pre-built store paths; + the difference is that it doesn't require the user to configure + binary caches and trusted public keys. diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc new file mode 100644 index 000000000..47f40ef33 --- /dev/null +++ b/src/libexpr/primops/fetchClosure.cc @@ -0,0 +1,152 @@ +#include "primops.hh" +#include "store-api.hh" +#include "make-content-addressed.hh" +#include "url.hh" + +namespace nix { + +static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature(Xp::FetchClosure); + + state.forceAttrs(*args[0], pos); + + std::optional fromStoreUrl; + std::optional fromPath; + bool toCA = false; + std::optional toPath; + + for (auto & attr : *args[0]->attrs) { + if (attr.name == "fromPath") { + PathSet context; + fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + + else if (attr.name == "toPath") { + state.forceValue(*attr.value, *attr.pos); + toCA = true; + if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + PathSet context; + toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + } + + else if (attr.name == "fromStore") + fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); + + else + throw Error({ + .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name), + .errPos = pos + }); + } + + if (!fromPath) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), + .errPos = pos + }); + + if (!fromStoreUrl) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), + .errPos = pos + }); + + auto parsedURL = parseURL(*fromStoreUrl); + + if (parsedURL.scheme != "http" && + parsedURL.scheme != "https" && + !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) + throw Error({ + .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), + .errPos = pos + }); + + auto fromStore = openStore(parsedURL.to_string()); + + if (toCA) { + if (!toPath || !state.store->isValidPath(*toPath)) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } + } else { + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + toPath = fromPath; + } + + /* In pure mode, require a CA path. */ + if (evalSettings.pureEval) { + auto info = state.store->queryPathInfo(*toPath); + if (!info->isContentAddressed(*state.store)) + throw Error({ + .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", + state.store->printStorePath(*toPath)), + .errPos = pos + }); + } + + auto toPathS = state.store->printStorePath(*toPath); + v.mkString(toPathS, {toPathS}); +} + +static RegisterPrimOp primop_fetchClosure({ + .name = "__fetchClosure", + .args = {"args"}, + .doc = R"( + Fetch a Nix store closure from a binary cache, rewriting it into + content-addressed form. For example, + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + fetches `/nix/store/r2jd...` from the specified binary cache, + and rewrites it into the content-addressed store path + `/nix/store/ldbh...`. + + If `fromPath` is already content-addressed, or if you are + allowing impure evaluation (`--impure`), then `toPath` may be + omitted. + + To find out the correct value for `toPath` given a `fromPath`, + you can use `nix store make-content-addressed`: + + ```console + # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 + rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' + ``` + + This function is similar to `builtins.storePath` in that it + allows you to use a previously built store path in a Nix + expression. However, it is more reproducible because it requires + specifying a binary cache from which the path can be fetched. + Also, requiring a content-addressed final store path avoids the + need for users to configure binary cache public keys. + )", + .fun = prim_fetchClosure, +}); + +} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9c2da2178..42c98e312 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -145,7 +145,7 @@ static void fetchTree( if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"), .errPos = pos }); @@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({ .fun = prim_fetchTarball, }); -static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc new file mode 100644 index 000000000..64d172918 --- /dev/null +++ b/src/libstore/make-content-addressed.cc @@ -0,0 +1,80 @@ +#include "make-content-addressed.hh" +#include "references.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths) +{ + StorePathSet closure; + srcStore.computeFSClosure(storePaths, closure); + + auto paths = srcStore.topoSortPaths(closure); + + std::reverse(paths.begin(), paths.end()); + + std::map remappings; + + for (auto & path : paths) { + auto pathS = srcStore.printStorePath(path); + auto oldInfo = srcStore.queryPathInfo(path); + std::string oldHashPart(path.hashPart()); + + StringSink sink; + srcStore.narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + sink.s = rewriteStrings(sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink(sink.s); + + auto narModuloHash = hashModuloSink.finish().first; + + auto dstPath = dstStore.makeFixedOutputPath( + FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + + printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + + StringSink sink2; + RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + rsink2(sink.s); + rsink2.flush(); + + ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path); + info.narSize = sink.s.size(); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }; + + StringSource source(sink2.s); + dstStore.addToStore(info, source); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + + return remappings; +} + +} diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh new file mode 100644 index 000000000..c4a66ed41 --- /dev/null +++ b/src/libstore/make-content-addressed.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths); + +} diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b49f47e1d..01f318fa3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -11,6 +11,7 @@ std::map stringifiedXpFeatures = { { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, + { Xp::FetchClosure, "fetch-closure" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 291a58e32..b5140dcfe 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -19,7 +19,8 @@ enum struct ExperimentalFeature Flakes, NixCommand, RecursiveNix, - NoUrlLiterals + NoUrlLiterals, + FetchClosure, }; /** diff --git a/src/nix/main.cc b/src/nix/main.cc index b923f2535..9bc6c15fa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"hash-path", {"hash", "path"}}, {"ls-nar", {"nar", "ls"}}, {"ls-store", {"store", "ls"}}, - {"make-content-addressable", {"store", "make-content-addressable"}}, + {"make-content-addressable", {"store", "make-content-addressed"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc deleted file mode 100644 index 2e75a3b61..000000000 --- a/src/nix/make-content-addressable.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include "command.hh" -#include "store-api.hh" -#include "references.hh" -#include "common-args.hh" -#include "json.hh" - -using namespace nix; - -struct CmdMakeContentAddressable : StorePathsCommand, MixJSON -{ - CmdMakeContentAddressable() - { - realiseMode = Realise::Outputs; - } - - std::string description() override - { - return "rewrite a path or closure to content-addressed form"; - } - - std::string doc() override - { - return - #include "make-content-addressable.md" - ; - } - - void run(ref store, StorePaths && storePaths) override - { - auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); - - std::reverse(paths.begin(), paths.end()); - - std::map remappings; - - auto jsonRoot = json ? std::make_unique(std::cout) : nullptr; - auto jsonRewrites = json ? std::make_unique(jsonRoot->object("rewrites")) : nullptr; - - for (auto & path : paths) { - auto pathS = store->printStorePath(path); - auto oldInfo = store->queryPathInfo(path); - std::string oldHashPart(path.hashPart()); - - StringSink sink; - store->narFromPath(path, sink); - - StringMap rewrites; - - StorePathSet references; - bool hasSelfReference = false; - for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } - } - - sink.s = rewriteStrings(sink.s, rewrites); - - HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink(sink.s); - - auto narHash = hashModuloSink.finish().first; - - ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); - info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; - - if (!json) - notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - store->addToStore(info, *source); - - if (json) - jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); - - remappings.insert_or_assign(std::move(path), std::move(info.path)); - } - } -}; - -static auto rCmdMakeContentAddressable = registerCommand2({"store", "make-content-addressable"}); diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc new file mode 100644 index 000000000..34860c38f --- /dev/null +++ b/src/nix/make-content-addressed.cc @@ -0,0 +1,55 @@ +#include "command.hh" +#include "store-api.hh" +#include "make-content-addressed.hh" +#include "common-args.hh" +#include "json.hh" + +using namespace nix; + +struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON +{ + CmdMakeContentAddressed() + { + realiseMode = Realise::Outputs; + } + + std::string description() override + { + return "rewrite a path or closure to content-addressed form"; + } + + std::string doc() override + { + return + #include "make-content-addressed.md" + ; + } + + void run(ref srcStore, StorePaths && storePaths) override + { + auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + + auto remappings = makeContentAddressed(*srcStore, *dstStore, + StorePathSet(storePaths.begin(), storePaths.end())); + + if (json) { + JSONObject jsonRoot(std::cout); + JSONObject jsonRewrites(jsonRoot.object("rewrites")); + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); + } + } else { + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + notice("rewrote '%s' to '%s'", + srcStore->printStorePath(path), + srcStore->printStorePath(i->second)); + } + } + } +}; + +static auto rCmdMakeContentAddressed = registerCommand2({"store", "make-content-addressed"}); diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressed.md similarity index 94% rename from src/nix/make-content-addressable.md rename to src/nix/make-content-addressed.md index 3dd847edc..215683e6d 100644 --- a/src/nix/make-content-addressable.md +++ b/src/nix/make-content-addressed.md @@ -5,7 +5,7 @@ R""( * Create a content-addressed representation of the closure of GNU Hello: ```console - # nix store make-content-addressable -r nixpkgs#hello + # nix store make-content-addressed nixpkgs#hello … rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' ``` @@ -29,7 +29,7 @@ R""( system closure: ```console - # nix store make-content-addressable -r /run/current-system + # nix store make-content-addressed /run/current-system ``` # Description diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 2368884f7..0361ac6a8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,6 +1,6 @@ source common.sh -needLocalStore "“--no-require-sigs” can’t be used with the daemon" +needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache clearStore diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh new file mode 100644 index 000000000..0c905ac43 --- /dev/null +++ b/tests/fetchClosure.sh @@ -0,0 +1,58 @@ +source common.sh + +enableFeatures "fetch-closure" +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +clearStore +clearCacheCache + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://$cacheDir $nonCaPath + +# Test basic fetchClosure rewriting from non-CA to CA. +clearStore + +[ ! -e $nonCaPath ] +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $caPath; + } +") = $caPath ]] + +[ ! -e $nonCaPath ] +[ -e $caPath ] + +# In impure mode, we can use non-CA paths. +[[ $(nix eval --raw --no-require-sigs --impure --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } +") = $nonCaPath ]] + +[ -e $nonCaPath ] + +# 'toPath' set to empty string should fail but print the expected path. +nix eval -v --json --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = \"\"; + } +" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" + +# If fromPath is CA, then toPath isn't needed. +nix copy --to file://$cacheDir $caPath + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + } +") = $caPath ]] diff --git a/tests/local.mk b/tests/local.mk index c686be049..97971dd76 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -96,7 +96,8 @@ nix_tests = \ describe-stores.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh + store-ping.sh \ + fetchClosure.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh