From f4bafc412fac79ce07c89f8d3ab9bd1c32f7b9cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Mar 2022 11:31:01 +0100 Subject: [PATCH 01/12] Add builtins.fetchClosure This allows closures to be imported at evaluation time, without requiring the user to configure substituters. E.g. builtins.fetchClosure { storePath = /nix/store/f89g6yi63m1ywfxj96whv5sxsm74w5ka-python3.9-sqlparse-0.4.2; from = "https://cache.ngi0.nixos.org"; } --- src/libexpr/primops/fetchClosure.cc | 62 +++++++++++++++++++++++++++++ src/libexpr/primops/fetchTree.cc | 4 +- 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 src/libexpr/primops/fetchClosure.cc diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc new file mode 100644 index 000000000..4bcc716db --- /dev/null +++ b/src/libexpr/primops/fetchClosure.cc @@ -0,0 +1,62 @@ +#include "primops.hh" +#include "store-api.hh" + +namespace nix { + +static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + std::optional storePath; + std::optional from; + + for (auto & attr : *args[0]->attrs) { + if (attr.name == "storePath") { + PathSet context; + storePath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + + else if (attr.name == "from") + from = state.forceStringNoCtx(*attr.value, *attr.pos); + + else + throw Error({ + .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name), + .errPos = pos + }); + } + + if (!storePath) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "storePath"), + .errPos = pos + }); + + if (!from) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "from"), + .errPos = pos + }); + + // FIXME: only allow some "trusted" store types (like BinaryCacheStore). + auto srcStore = openStore(*from); + + copyClosure(*srcStore, *state.store, RealisedPath::Set { *storePath }); + + auto storePathS = state.store->printStorePath(*storePath); + + v.mkString(storePathS, {storePathS}); + + // FIXME: in pure mode, require a CA path or a NAR hash of the + // top-level path. +} + +static RegisterPrimOp primop_fetchClosure({ + .name = "__fetchClosure", + .args = {"args"}, + .doc = R"( + )", + .fun = prim_fetchClosure, +}); + +} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9c2da2178..42c98e312 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -145,7 +145,7 @@ static void fetchTree( if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"), .errPos = pos }); @@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({ .fun = prim_fetchTarball, }); -static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } From 41659418cfeb7a33fc76716a1847f79ae13d9b0c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Mar 2022 14:00:54 +0100 Subject: [PATCH 02/12] fetchClosure: Require a CA path in pure mode --- src/libexpr/primops/fetchClosure.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 4bcc716db..22a4649a4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -43,12 +43,19 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args copyClosure(*srcStore, *state.store, RealisedPath::Set { *storePath }); + /* In pure mode, require a CA path. */ + if (evalSettings.pureEval) { + auto info = state.store->queryPathInfo(*storePath); + if (!info->isContentAddressed(*state.store)) + throw Error({ + .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", + state.store->printStorePath(*storePath)), + .errPos = pos + }); + } + auto storePathS = state.store->printStorePath(*storePath); - v.mkString(storePathS, {storePathS}); - - // FIXME: in pure mode, require a CA path or a NAR hash of the - // top-level path. } static RegisterPrimOp primop_fetchClosure({ From 7f6fe8ca1d41bceef32790aa0313aa62ae2b65fb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Mar 2022 14:34:45 +0100 Subject: [PATCH 03/12] Rename --- src/libexpr/primops/fetchClosure.cc | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 22a4649a4..8e60b2ccc 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -7,17 +7,17 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args { state.forceAttrs(*args[0], pos); - std::optional storePath; - std::optional from; + std::optional fromStoreUrl; + std::optional fromPath; for (auto & attr : *args[0]->attrs) { - if (attr.name == "storePath") { + if (attr.name == "fromPath") { PathSet context; - storePath = state.coerceToStorePath(*attr.pos, *attr.value, context); + fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); } - else if (attr.name == "from") - from = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (attr.name == "fromStore") + fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); else throw Error({ @@ -26,36 +26,36 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args }); } - if (!storePath) + if (!fromPath) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "storePath"), + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), .errPos = pos }); - if (!from) + if (!fromStoreUrl) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "from"), + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), .errPos = pos }); // FIXME: only allow some "trusted" store types (like BinaryCacheStore). - auto srcStore = openStore(*from); + auto fromStore = openStore(*fromStoreUrl); - copyClosure(*srcStore, *state.store, RealisedPath::Set { *storePath }); + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); /* In pure mode, require a CA path. */ if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*storePath); + auto info = state.store->queryPathInfo(*fromPath); if (!info->isContentAddressed(*state.store)) throw Error({ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*storePath)), + state.store->printStorePath(*fromPath)), .errPos = pos }); } - auto storePathS = state.store->printStorePath(*storePath); - v.mkString(storePathS, {storePathS}); + auto fromPathS = state.store->printStorePath(*fromPath); + v.mkString(fromPathS, {fromPathS}); } static RegisterPrimOp primop_fetchClosure({ From 545c2d0d8cbac86c169a6dd049c1ed9c3913774d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 21:14:58 +0100 Subject: [PATCH 04/12] fetchClosure: Allow a path to be rewritten to CA on the fly The advantage is that the resulting closure doesn't need to be signed, so you don't need to configure any binary cache keys on the client. --- src/libexpr/primops/fetchClosure.cc | 46 ++++++++++++-- src/libstore/make-content-addressed.cc | 79 ++++++++++++++++++++++++ src/libstore/make-content-addressed.hh | 12 ++++ src/nix/make-content-addressable.cc | 85 ++++++-------------------- 4 files changed, 150 insertions(+), 72 deletions(-) create mode 100644 src/libstore/make-content-addressed.cc create mode 100644 src/libstore/make-content-addressed.hh diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 8e60b2ccc..56fd53ed5 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "make-content-addressed.hh" namespace nix { @@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args std::optional fromStoreUrl; std::optional fromPath; + bool toCA = false; + std::optional toPath; for (auto & attr : *args[0]->attrs) { if (attr.name == "fromPath") { @@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); } + else if (attr.name == "toPath") { + state.forceValue(*attr.value, *attr.pos); + toCA = true; + if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + PathSet context; + toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + } + else if (attr.name == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); @@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args // FIXME: only allow some "trusted" store types (like BinaryCacheStore). auto fromStore = openStore(*fromStoreUrl); - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + if (toCA) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } else { + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + toPath = fromPath; + } /* In pure mode, require a CA path. */ if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*fromPath); + auto info = state.store->queryPathInfo(*toPath); if (!info->isContentAddressed(*state.store)) throw Error({ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*fromPath)), + state.store->printStorePath(*toPath)), .errPos = pos }); } - auto fromPathS = state.store->printStorePath(*fromPath); - v.mkString(fromPathS, {fromPathS}); + auto toPathS = state.store->printStorePath(*toPath); + v.mkString(toPathS, {toPathS}); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc new file mode 100644 index 000000000..0b95ff37c --- /dev/null +++ b/src/libstore/make-content-addressed.cc @@ -0,0 +1,79 @@ +#include "make-content-addressed.hh" +#include "references.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths) +{ + // FIXME: use closure of storePaths. + + auto paths = srcStore.topoSortPaths(storePaths); + + std::reverse(paths.begin(), paths.end()); + + std::map remappings; + + for (auto & path : paths) { + auto pathS = srcStore.printStorePath(path); + auto oldInfo = srcStore.queryPathInfo(path); + std::string oldHashPart(path.hashPart()); + + StringSink sink; + srcStore.narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + sink.s = rewriteStrings(sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink(sink.s); + + auto narHash = hashModuloSink.finish().first; + + ValidPathInfo info { + dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path); + info.narSize = sink.s.size(); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + }; + + printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path)); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); + rsink2(sink.s); + rsink2.flush(); + }); + + dstStore.addToStore(info, *source); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + + return remappings; +} + +} diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh new file mode 100644 index 000000000..c4a66ed41 --- /dev/null +++ b/src/libstore/make-content-addressed.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths); + +} diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 2e75a3b61..a8579ea7c 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "store-api.hh" -#include "references.hh" +#include "make-content-addressed.hh" #include "common-args.hh" #include "json.hh" @@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON void run(ref store, StorePaths && storePaths) override { - auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); + auto remappings = makeContentAddressed(*store, *store, + StorePathSet(storePaths.begin(), storePaths.end())); - std::reverse(paths.begin(), paths.end()); - - std::map remappings; - - auto jsonRoot = json ? std::make_unique(std::cout) : nullptr; - auto jsonRewrites = json ? std::make_unique(jsonRoot->object("rewrites")) : nullptr; - - for (auto & path : paths) { - auto pathS = store->printStorePath(path); - auto oldInfo = store->queryPathInfo(path); - std::string oldHashPart(path.hashPart()); - - StringSink sink; - store->narFromPath(path, sink); - - StringMap rewrites; - - StorePathSet references; - bool hasSelfReference = false; - for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + if (json) { + JSONObject jsonRoot(std::cout); + JSONObject jsonRewrites(jsonRoot.object("rewrites")); + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second)); + } + } else { + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + notice("rewrote '%s' to '%s'", + store->printStorePath(path), + store->printStorePath(i->second)); } - - sink.s = rewriteStrings(sink.s, rewrites); - - HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink(sink.s); - - auto narHash = hashModuloSink.finish().first; - - ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); - info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; - - if (!json) - notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - store->addToStore(info, *source); - - if (json) - jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); - - remappings.insert_or_assign(std::move(path), std::move(info.path)); } } }; From f18607549ce38545b1d754ed93f3b7c5417970d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 21:47:50 +0100 Subject: [PATCH 05/12] Fix makeContentAddressed() on self-references LocalStore::addToStore() since 79ae9e4558cbefd743f28a5e73110c2303b03a85 expects a regular NAR hash, rather than a NAR hash modulo self-references. Fixes #6300. Also, makeContentAddressed() now rewrites the entire closure (so 'nix store make-content-addressable' no longer needs '-r'). See #6301. --- src/libstore/make-content-addressed.cc | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 0b95ff37c..fc11fcb27 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -8,9 +8,10 @@ std::map makeContentAddressed( Store & dstStore, const StorePathSet & storePaths) { - // FIXME: use closure of storePaths. + StorePathSet closure; + srcStore.computeFSClosure(storePaths, closure); - auto paths = srcStore.topoSortPaths(storePaths); + auto paths = srcStore.topoSortPaths(closure); std::reverse(paths.begin(), paths.end()); @@ -46,29 +47,29 @@ std::map makeContentAddressed( HashModuloSink hashModuloSink(htSHA256, oldHashPart); hashModuloSink(sink.s); - auto narHash = hashModuloSink.finish().first; + auto narModuloHash = hashModuloSink.finish().first; - ValidPathInfo info { - dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; + auto dstPath = dstStore.makeFixedOutputPath( + FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + + printInfo("rewroting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + + StringSink sink2; + RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + rsink2(sink.s); + rsink2.flush(); + + ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path); info.narSize = sink.s.size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, - .hash = info.narHash, + .hash = narModuloHash, }; - printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - dstStore.addToStore(info, *source); + StringSource source(sink2.s); + dstStore.addToStore(info, source); remappings.insert_or_assign(std::move(path), std::move(info.path)); } From 5acaf13d3564f689e5461f29a9cc5958809d5e93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 21:54:49 +0100 Subject: [PATCH 06/12] Rename 'nix store make-content-addressable' to 'nix store make-content-addressed' --- doc/manual/src/release-notes/rl-next.md | 3 +++ src/nix/main.cc | 2 +- ...e-content-addressable.cc => make-content-addressed.cc} | 8 ++++---- ...e-content-addressable.md => make-content-addressed.md} | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) rename src/nix/{make-content-addressable.cc => make-content-addressed.cc} (83%) rename src/nix/{make-content-addressable.md => make-content-addressed.md} (94%) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c9753f9aa..8fbc605e7 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,6 @@ # Release X.Y (202?-??-??) * Various nix commands can now read expressions from stdin with `--file -`. + +* `nix store make-content-addressable` has been renamed to `nix store + make-content-addressed`. diff --git a/src/nix/main.cc b/src/nix/main.cc index b923f2535..9bc6c15fa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"hash-path", {"hash", "path"}}, {"ls-nar", {"nar", "ls"}}, {"ls-store", {"store", "ls"}}, - {"make-content-addressable", {"store", "make-content-addressable"}}, + {"make-content-addressable", {"store", "make-content-addressed"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressed.cc similarity index 83% rename from src/nix/make-content-addressable.cc rename to src/nix/make-content-addressed.cc index a8579ea7c..dc0447cb8 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressed.cc @@ -6,9 +6,9 @@ using namespace nix; -struct CmdMakeContentAddressable : StorePathsCommand, MixJSON +struct CmdMakeContentAddressed : StorePathsCommand, MixJSON { - CmdMakeContentAddressable() + CmdMakeContentAddressed() { realiseMode = Realise::Outputs; } @@ -21,7 +21,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON std::string doc() override { return - #include "make-content-addressable.md" + #include "make-content-addressed.md" ; } @@ -50,4 +50,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto rCmdMakeContentAddressable = registerCommand2({"store", "make-content-addressable"}); +static auto rCmdMakeContentAddressed = registerCommand2({"store", "make-content-addressed"}); diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressed.md similarity index 94% rename from src/nix/make-content-addressable.md rename to src/nix/make-content-addressed.md index 3dd847edc..215683e6d 100644 --- a/src/nix/make-content-addressable.md +++ b/src/nix/make-content-addressed.md @@ -5,7 +5,7 @@ R""( * Create a content-addressed representation of the closure of GNU Hello: ```console - # nix store make-content-addressable -r nixpkgs#hello + # nix store make-content-addressed nixpkgs#hello … rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' ``` @@ -29,7 +29,7 @@ R""( system closure: ```console - # nix store make-content-addressable -r /run/current-system + # nix store make-content-addressed /run/current-system ``` # Description From 7ffda0af6effbf32c8668f34cc3f0448c58bc3c1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 22:01:20 +0100 Subject: [PATCH 07/12] fetchClosure: Skip makeContentAddressed() if toPath is already valid --- src/libexpr/primops/fetchClosure.cc | 42 +++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 56fd53ed5..c3f07b6d6 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -54,26 +54,28 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args auto fromStore = openStore(*fromStoreUrl); if (toCA) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), - state.store->printStorePath(*toPath)), - .errPos = pos - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), - .errPos = pos - }); + if (!toPath || !state.store->isValidPath(*toPath)) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } } else { copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); toPath = fromPath; From 4120930ac19ab7296818fdc1d1389e7799168867 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 22:47:33 +0100 Subject: [PATCH 08/12] fetchClosure: Only allow some "safe" store types --- src/libexpr/primops/fetchClosure.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index c3f07b6d6..247bceb07 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,6 +1,7 @@ #include "primops.hh" #include "store-api.hh" #include "make-content-addressed.hh" +#include "url.hh" namespace nix { @@ -50,8 +51,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - // FIXME: only allow some "trusted" store types (like BinaryCacheStore). - auto fromStore = openStore(*fromStoreUrl); + auto parsedURL = parseURL(*fromStoreUrl); + + if (parsedURL.scheme != "http" && parsedURL.scheme != "https") + throw Error({ + .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), + .errPos = pos + }); + + auto fromStore = openStore(parsedURL.to_string()); if (toCA) { if (!toPath || !state.store->isValidPath(*toPath)) { From 28186b7044dca513e6e07c3e66b7de2143543ae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 23:19:21 +0100 Subject: [PATCH 09/12] Add a test for fetchClosure and 'nix store make-content-addressed' --- src/libexpr/primops/fetchClosure.cc | 4 +- src/libstore/make-content-addressed.cc | 2 +- tests/binary-cache.sh | 2 +- tests/fetchClosure.sh | 57 ++++++++++++++++++++++++++ tests/local.mk | 3 +- 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/fetchClosure.sh diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 247bceb07..47e2d2bf2 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -53,7 +53,9 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args auto parsedURL = parseURL(*fromStoreUrl); - if (parsedURL.scheme != "http" && parsedURL.scheme != "https") + if (parsedURL.scheme != "http" && + parsedURL.scheme != "https" && + !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) throw Error({ .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), .errPos = pos diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index fc11fcb27..64d172918 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( auto dstPath = dstStore.makeFixedOutputPath( FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); - printInfo("rewroting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); StringSink sink2; RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 2368884f7..0361ac6a8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,6 +1,6 @@ source common.sh -needLocalStore "“--no-require-sigs” can’t be used with the daemon" +needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache clearStore diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh new file mode 100644 index 000000000..811d44af9 --- /dev/null +++ b/tests/fetchClosure.sh @@ -0,0 +1,57 @@ +source common.sh + +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +clearStore +clearCacheCache + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://$cacheDir $nonCaPath + +# Test basic fetchClosure rewriting from non-CA to CA. +clearStore + +[ ! -e $nonCaPath ] +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $caPath; + } +") = $caPath ]] + +[ ! -e $nonCaPath ] +[ -e $caPath ] + +# In impure mode, we can use non-CA paths. +[[ $(nix eval --raw --no-require-sigs --impure --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } +") = $nonCaPath ]] + +[ -e $nonCaPath ] + +# 'toPath' set to empty string should fail but print the expected path. +nix eval -v --json --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = \"\"; + } +" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" + +# If fromPath is CA, then toPath isn't needed. +nix copy --to file://$cacheDir $caPath + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + } +") = $caPath ]] diff --git a/tests/local.mk b/tests/local.mk index c686be049..97971dd76 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -96,7 +96,8 @@ nix_tests = \ describe-stores.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh + store-ping.sh \ + fetchClosure.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From 98658ae9d25e5e715f9349fbbb79d0ba31bb810c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 23:31:48 +0100 Subject: [PATCH 10/12] Document fetchClosure --- doc/manual/src/release-notes/rl-next.md | 7 ++++++ src/libexpr/primops/fetchClosure.cc | 33 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8fbc605e7..33eaa8a2c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,3 +4,10 @@ * `nix store make-content-addressable` has been renamed to `nix store make-content-addressed`. + +* New builtin function `builtins.fetchClosure` that copies a closure + from a binary cache at evaluation time and rewrites it to + content-addressed form (if it isn't already). Like + `builtins.storePath`, this allows importing pre-built store paths; + the difference is that it doesn't require the user to configure + binary caches and trusted public keys. diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 47e2d2bf2..045e97dbd 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -110,6 +110,39 @@ static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( + Fetch a Nix store closure from a binary cache, rewriting it into + content-addressed form. For example, + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + fetches `/nix/store/r2jd...` from the specified binary cache, + and rewrites it into the content-addressed store path + `/nix/store/ldbh...`. + + If `fromPath` is already content-addressed, or if you are + allowing impure evaluation (`--impure`), then `toPath` may be + omitted. + + To find out the correct value for `toPath` given a `fromPath`, + you can use `nix store make-content-addressed`: + + ```console + # nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 + rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' + ``` + + This function is similar to `builtins.storePath` in that it + allows you to use a previously built store path in a Nix + expression. However, it is more reproducible because it requires + specifying a binary cache from which the path can be fetched. + Also, requiring a content-addressed final store path avoids the + need for users to configure binary cache public keys. )", .fun = prim_fetchClosure, }); From e5f7029ba49a486c1b0c8011be79b8b41be20935 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 23:40:04 +0100 Subject: [PATCH 11/12] nix store make-content-addressed: Support --from / --to --- src/libexpr/primops/fetchClosure.cc | 2 +- src/nix/make-content-addressed.cc | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 045e97dbd..3e07d1d18 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -133,7 +133,7 @@ static RegisterPrimOp primop_fetchClosure({ you can use `nix store make-content-addressed`: ```console - # nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 + # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index dc0447cb8..34860c38f 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -6,7 +6,7 @@ using namespace nix; -struct CmdMakeContentAddressed : StorePathsCommand, MixJSON +struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON { CmdMakeContentAddressed() { @@ -25,9 +25,11 @@ struct CmdMakeContentAddressed : StorePathsCommand, MixJSON ; } - void run(ref store, StorePaths && storePaths) override + void run(ref srcStore, StorePaths && storePaths) override { - auto remappings = makeContentAddressed(*store, *store, + auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + + auto remappings = makeContentAddressed(*srcStore, *dstStore, StorePathSet(storePaths.begin(), storePaths.end())); if (json) { @@ -36,15 +38,15 @@ struct CmdMakeContentAddressed : StorePathsCommand, MixJSON for (auto & path : storePaths) { auto i = remappings.find(path); assert(i != remappings.end()); - jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second)); + jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); } } else { for (auto & path : storePaths) { auto i = remappings.find(path); assert(i != remappings.end()); notice("rewrote '%s' to '%s'", - store->printStorePath(path), - store->printStorePath(i->second)); + srcStore->printStorePath(path), + srcStore->printStorePath(i->second)); } } } From f902f3c2cbfa1a78bec8d135297abe244452e794 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 21:33:20 +0100 Subject: [PATCH 12/12] Add experimental feature 'fetch-closure' --- src/libexpr/primops/fetchClosure.cc | 2 ++ src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 3 ++- tests/fetchClosure.sh | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 3e07d1d18..47f40ef33 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -7,6 +7,8 @@ namespace nix { static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { + settings.requireExperimentalFeature(Xp::FetchClosure); + state.forceAttrs(*args[0], pos); std::optional fromStoreUrl; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b49f47e1d..01f318fa3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -11,6 +11,7 @@ std::map stringifiedXpFeatures = { { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, + { Xp::FetchClosure, "fetch-closure" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 291a58e32..b5140dcfe 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -19,7 +19,8 @@ enum struct ExperimentalFeature Flakes, NixCommand, RecursiveNix, - NoUrlLiterals + NoUrlLiterals, + FetchClosure, }; /** diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 811d44af9..0c905ac43 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -1,5 +1,6 @@ source common.sh +enableFeatures "fetch-closure" needLocalStore "'--no-require-sigs' can’t be used with the daemon" clearStore