Merge pull request #6302 from edolstra/fetch-closure
Add builtins.fetchClosure
This commit is contained in:
Eelco Dolstra
GitHub
commit
d9cfd853e5
|
|
@ -1,3 +1,13 @@
|
|||
# Release X.Y (202?-??-??)
|
||||
|
||||
* Various nix commands can now read expressions from stdin with `--file -`.
|
||||
|
||||
* `nix store make-content-addressable` has been renamed to `nix store
|
||||
make-content-addressed`.
|
||||
|
||||
* New builtin function `builtins.fetchClosure` that copies a closure
|
||||
from a binary cache at evaluation time and rewrites it to
|
||||
content-addressed form (if it isn't already). Like
|
||||
`builtins.storePath`, this allows importing pre-built store paths;
|
||||
the difference is that it doesn't require the user to configure
|
||||
binary caches and trusted public keys.
|
||||
|
|
|
|||
152
src/libexpr/primops/fetchClosure.cc
Normal file
152
src/libexpr/primops/fetchClosure.cc
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
#include "primops.hh"
|
||||
#include "store-api.hh"
|
||||
#include "make-content-addressed.hh"
|
||||
#include "url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
settings.requireExperimentalFeature(Xp::FetchClosure);
|
||||
|
||||
state.forceAttrs(*args[0], pos);
|
||||
|
||||
std::optional<std::string> fromStoreUrl;
|
||||
std::optional<StorePath> fromPath;
|
||||
bool toCA = false;
|
||||
std::optional<StorePath> toPath;
|
||||
|
||||
for (auto & attr : *args[0]->attrs) {
|
||||
if (attr.name == "fromPath") {
|
||||
PathSet context;
|
||||
fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
|
||||
}
|
||||
|
||||
else if (attr.name == "toPath") {
|
||||
state.forceValue(*attr.value, *attr.pos);
|
||||
toCA = true;
|
||||
if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
|
||||
PathSet context;
|
||||
toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
|
||||
}
|
||||
}
|
||||
|
||||
else if (attr.name == "fromStore")
|
||||
fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||
|
||||
else
|
||||
throw Error({
|
||||
.msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
if (!fromPath)
|
||||
throw Error({
|
||||
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
if (!fromStoreUrl)
|
||||
throw Error({
|
||||
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
auto parsedURL = parseURL(*fromStoreUrl);
|
||||
|
||||
if (parsedURL.scheme != "http" &&
|
||||
parsedURL.scheme != "https" &&
|
||||
!(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
|
||||
throw Error({
|
||||
.msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
auto fromStore = openStore(parsedURL.to_string());
|
||||
|
||||
if (toCA) {
|
||||
if (!toPath || !state.store->isValidPath(*toPath)) {
|
||||
auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
|
||||
auto i = remappings.find(*fromPath);
|
||||
assert(i != remappings.end());
|
||||
if (toPath && *toPath != i->second)
|
||||
throw Error({
|
||||
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
|
||||
state.store->printStorePath(*fromPath),
|
||||
state.store->printStorePath(i->second),
|
||||
state.store->printStorePath(*toPath)),
|
||||
.errPos = pos
|
||||
});
|
||||
if (!toPath)
|
||||
throw Error({
|
||||
.msg = hintfmt(
|
||||
"rewriting '%s' to content-addressed form yielded '%s'; "
|
||||
"please set this in the 'toPath' attribute passed to 'fetchClosure'",
|
||||
state.store->printStorePath(*fromPath),
|
||||
state.store->printStorePath(i->second)),
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
} else {
|
||||
copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
|
||||
toPath = fromPath;
|
||||
}
|
||||
|
||||
/* In pure mode, require a CA path. */
|
||||
if (evalSettings.pureEval) {
|
||||
auto info = state.store->queryPathInfo(*toPath);
|
||||
if (!info->isContentAddressed(*state.store))
|
||||
throw Error({
|
||||
.msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
|
||||
state.store->printStorePath(*toPath)),
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
auto toPathS = state.store->printStorePath(*toPath);
|
||||
v.mkString(toPathS, {toPathS});
|
||||
}
|
||||
|
||||
static RegisterPrimOp primop_fetchClosure({
|
||||
.name = "__fetchClosure",
|
||||
.args = {"args"},
|
||||
.doc = R"(
|
||||
Fetch a Nix store closure from a binary cache, rewriting it into
|
||||
content-addressed form. For example,
|
||||
|
||||
```nix
|
||||
builtins.fetchClosure {
|
||||
fromStore = "https://cache.nixos.org";
|
||||
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
|
||||
toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
|
||||
}
|
||||
```
|
||||
|
||||
fetches `/nix/store/r2jd...` from the specified binary cache,
|
||||
and rewrites it into the content-addressed store path
|
||||
`/nix/store/ldbh...`.
|
||||
|
||||
If `fromPath` is already content-addressed, or if you are
|
||||
allowing impure evaluation (`--impure`), then `toPath` may be
|
||||
omitted.
|
||||
|
||||
To find out the correct value for `toPath` given a `fromPath`,
|
||||
you can use `nix store make-content-addressed`:
|
||||
|
||||
```console
|
||||
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
|
||||
rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
|
||||
```
|
||||
|
||||
This function is similar to `builtins.storePath` in that it
|
||||
allows you to use a previously built store path in a Nix
|
||||
expression. However, it is more reproducible because it requires
|
||||
specifying a binary cache from which the path can be fetched.
|
||||
Also, requiring a content-addressed final store path avoids the
|
||||
need for users to configure binary cache public keys.
|
||||
)",
|
||||
.fun = prim_fetchClosure,
|
||||
});
|
||||
|
||||
}
|
||||
|
|
@ -145,7 +145,7 @@ static void fetchTree(
|
|||
if (!params.allowNameArgument)
|
||||
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
|
||||
throw Error({
|
||||
.msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
|
||||
.msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
|
|
|
|||
80
src/libstore/make-content-addressed.cc
Normal file
80
src/libstore/make-content-addressed.cc
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
#include "make-content-addressed.hh"
|
||||
#include "references.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::map<StorePath, StorePath> makeContentAddressed(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const StorePathSet & storePaths)
|
||||
{
|
||||
StorePathSet closure;
|
||||
srcStore.computeFSClosure(storePaths, closure);
|
||||
|
||||
auto paths = srcStore.topoSortPaths(closure);
|
||||
|
||||
std::reverse(paths.begin(), paths.end());
|
||||
|
||||
std::map<StorePath, StorePath> remappings;
|
||||
|
||||
for (auto & path : paths) {
|
||||
auto pathS = srcStore.printStorePath(path);
|
||||
auto oldInfo = srcStore.queryPathInfo(path);
|
||||
std::string oldHashPart(path.hashPart());
|
||||
|
||||
StringSink sink;
|
||||
srcStore.narFromPath(path, sink);
|
||||
|
||||
StringMap rewrites;
|
||||
|
||||
StorePathSet references;
|
||||
bool hasSelfReference = false;
|
||||
for (auto & ref : oldInfo->references) {
|
||||
if (ref == path)
|
||||
hasSelfReference = true;
|
||||
else {
|
||||
auto i = remappings.find(ref);
|
||||
auto replacement = i != remappings.end() ? i->second : ref;
|
||||
// FIXME: warn about unremapped paths?
|
||||
if (replacement != ref)
|
||||
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
|
||||
references.insert(std::move(replacement));
|
||||
}
|
||||
}
|
||||
|
||||
sink.s = rewriteStrings(sink.s, rewrites);
|
||||
|
||||
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
|
||||
hashModuloSink(sink.s);
|
||||
|
||||
auto narModuloHash = hashModuloSink.finish().first;
|
||||
|
||||
auto dstPath = dstStore.makeFixedOutputPath(
|
||||
FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
|
||||
|
||||
printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
|
||||
|
||||
StringSink sink2;
|
||||
RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
|
||||
rsink2(sink.s);
|
||||
rsink2.flush();
|
||||
|
||||
ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
|
||||
info.references = std::move(references);
|
||||
if (hasSelfReference) info.references.insert(info.path);
|
||||
info.narSize = sink.s.size();
|
||||
info.ca = FixedOutputHash {
|
||||
.method = FileIngestionMethod::Recursive,
|
||||
.hash = narModuloHash,
|
||||
};
|
||||
|
||||
StringSource source(sink2.s);
|
||||
dstStore.addToStore(info, source);
|
||||
|
||||
remappings.insert_or_assign(std::move(path), std::move(info.path));
|
||||
}
|
||||
|
||||
return remappings;
|
||||
}
|
||||
|
||||
}
|
||||
12
src/libstore/make-content-addressed.hh
Normal file
12
src/libstore/make-content-addressed.hh
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
#pragma once
|
||||
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::map<StorePath, StorePath> makeContentAddressed(
|
||||
Store & srcStore,
|
||||
Store & dstStore,
|
||||
const StorePathSet & storePaths);
|
||||
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
|
|||
{ Xp::NixCommand, "nix-command" },
|
||||
{ Xp::RecursiveNix, "recursive-nix" },
|
||||
{ Xp::NoUrlLiterals, "no-url-literals" },
|
||||
{ Xp::FetchClosure, "fetch-closure" },
|
||||
};
|
||||
|
||||
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@ enum struct ExperimentalFeature
|
|||
Flakes,
|
||||
NixCommand,
|
||||
RecursiveNix,
|
||||
NoUrlLiterals
|
||||
NoUrlLiterals,
|
||||
FetchClosure,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
|
|||
{"hash-path", {"hash", "path"}},
|
||||
{"ls-nar", {"nar", "ls"}},
|
||||
{"ls-store", {"store", "ls"}},
|
||||
{"make-content-addressable", {"store", "make-content-addressable"}},
|
||||
{"make-content-addressable", {"store", "make-content-addressed"}},
|
||||
{"optimise-store", {"store", "optimise"}},
|
||||
{"ping-store", {"store", "ping"}},
|
||||
{"sign-paths", {"store", "sign"}},
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
#include "command.hh"
|
||||
#include "store-api.hh"
|
||||
#include "references.hh"
|
||||
#include "common-args.hh"
|
||||
#include "json.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
|
||||
{
|
||||
CmdMakeContentAddressable()
|
||||
{
|
||||
realiseMode = Realise::Outputs;
|
||||
}
|
||||
|
||||
std::string description() override
|
||||
{
|
||||
return "rewrite a path or closure to content-addressed form";
|
||||
}
|
||||
|
||||
std::string doc() override
|
||||
{
|
||||
return
|
||||
#include "make-content-addressable.md"
|
||||
;
|
||||
}
|
||||
|
||||
void run(ref<Store> store, StorePaths && storePaths) override
|
||||
{
|
||||
auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
|
||||
|
||||
std::reverse(paths.begin(), paths.end());
|
||||
|
||||
std::map<StorePath, StorePath> remappings;
|
||||
|
||||
auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
|
||||
auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
|
||||
|
||||
for (auto & path : paths) {
|
||||
auto pathS = store->printStorePath(path);
|
||||
auto oldInfo = store->queryPathInfo(path);
|
||||
std::string oldHashPart(path.hashPart());
|
||||
|
||||
StringSink sink;
|
||||
store->narFromPath(path, sink);
|
||||
|
||||
StringMap rewrites;
|
||||
|
||||
StorePathSet references;
|
||||
bool hasSelfReference = false;
|
||||
for (auto & ref : oldInfo->references) {
|
||||
if (ref == path)
|
||||
hasSelfReference = true;
|
||||
else {
|
||||
auto i = remappings.find(ref);
|
||||
auto replacement = i != remappings.end() ? i->second : ref;
|
||||
// FIXME: warn about unremapped paths?
|
||||
if (replacement != ref)
|
||||
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
|
||||
references.insert(std::move(replacement));
|
||||
}
|
||||
}
|
||||
|
||||
sink.s = rewriteStrings(sink.s, rewrites);
|
||||
|
||||
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
|
||||
hashModuloSink(sink.s);
|
||||
|
||||
auto narHash = hashModuloSink.finish().first;
|
||||
|
||||
ValidPathInfo info {
|
||||
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
|
||||
narHash,
|
||||
};
|
||||
info.references = std::move(references);
|
||||
if (hasSelfReference) info.references.insert(info.path);
|
||||
info.narSize = sink.s.size();
|
||||
info.ca = FixedOutputHash {
|
||||
.method = FileIngestionMethod::Recursive,
|
||||
.hash = info.narHash,
|
||||
};
|
||||
|
||||
if (!json)
|
||||
notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
|
||||
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
|
||||
rsink2(sink.s);
|
||||
rsink2.flush();
|
||||
});
|
||||
|
||||
store->addToStore(info, *source);
|
||||
|
||||
if (json)
|
||||
jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
|
||||
|
||||
remappings.insert_or_assign(std::move(path), std::move(info.path));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});
|
||||
55
src/nix/make-content-addressed.cc
Normal file
55
src/nix/make-content-addressed.cc
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
#include "command.hh"
|
||||
#include "store-api.hh"
|
||||
#include "make-content-addressed.hh"
|
||||
#include "common-args.hh"
|
||||
#include "json.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
|
||||
{
|
||||
CmdMakeContentAddressed()
|
||||
{
|
||||
realiseMode = Realise::Outputs;
|
||||
}
|
||||
|
||||
std::string description() override
|
||||
{
|
||||
return "rewrite a path or closure to content-addressed form";
|
||||
}
|
||||
|
||||
std::string doc() override
|
||||
{
|
||||
return
|
||||
#include "make-content-addressed.md"
|
||||
;
|
||||
}
|
||||
|
||||
void run(ref<Store> srcStore, StorePaths && storePaths) override
|
||||
{
|
||||
auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
|
||||
|
||||
auto remappings = makeContentAddressed(*srcStore, *dstStore,
|
||||
StorePathSet(storePaths.begin(), storePaths.end()));
|
||||
|
||||
if (json) {
|
||||
JSONObject jsonRoot(std::cout);
|
||||
JSONObject jsonRewrites(jsonRoot.object("rewrites"));
|
||||
for (auto & path : storePaths) {
|
||||
auto i = remappings.find(path);
|
||||
assert(i != remappings.end());
|
||||
jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
|
||||
}
|
||||
} else {
|
||||
for (auto & path : storePaths) {
|
||||
auto i = remappings.find(path);
|
||||
assert(i != remappings.end());
|
||||
notice("rewrote '%s' to '%s'",
|
||||
srcStore->printStorePath(path),
|
||||
srcStore->printStorePath(i->second));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});
|
||||
|
|
@ -5,7 +5,7 @@ R""(
|
|||
* Create a content-addressed representation of the closure of GNU Hello:
|
||||
|
||||
```console
|
||||
# nix store make-content-addressable -r nixpkgs#hello
|
||||
# nix store make-content-addressed nixpkgs#hello
|
||||
…
|
||||
rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
|
||||
```
|
||||
|
|
@ -29,7 +29,7 @@ R""(
|
|||
system closure:
|
||||
|
||||
```console
|
||||
# nix store make-content-addressable -r /run/current-system
|
||||
# nix store make-content-addressed /run/current-system
|
||||
```
|
||||
|
||||
# Description
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
source common.sh
|
||||
|
||||
needLocalStore "“--no-require-sigs” can’t be used with the daemon"
|
||||
needLocalStore "'--no-require-sigs' can’t be used with the daemon"
|
||||
|
||||
# We can produce drvs directly into the binary cache
|
||||
clearStore
|
||||
|
|
|
|||
58
tests/fetchClosure.sh
Normal file
58
tests/fetchClosure.sh
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
source common.sh
|
||||
|
||||
enableFeatures "fetch-closure"
|
||||
needLocalStore "'--no-require-sigs' can’t be used with the daemon"
|
||||
|
||||
clearStore
|
||||
clearCacheCache
|
||||
|
||||
# Initialize binary cache.
|
||||
nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
|
||||
caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
|
||||
nix copy --to file://$cacheDir $nonCaPath
|
||||
|
||||
# Test basic fetchClosure rewriting from non-CA to CA.
|
||||
clearStore
|
||||
|
||||
[ ! -e $nonCaPath ]
|
||||
[ ! -e $caPath ]
|
||||
|
||||
[[ $(nix eval -v --raw --expr "
|
||||
builtins.fetchClosure {
|
||||
fromStore = \"file://$cacheDir\";
|
||||
fromPath = $nonCaPath;
|
||||
toPath = $caPath;
|
||||
}
|
||||
") = $caPath ]]
|
||||
|
||||
[ ! -e $nonCaPath ]
|
||||
[ -e $caPath ]
|
||||
|
||||
# In impure mode, we can use non-CA paths.
|
||||
[[ $(nix eval --raw --no-require-sigs --impure --expr "
|
||||
builtins.fetchClosure {
|
||||
fromStore = \"file://$cacheDir\";
|
||||
fromPath = $nonCaPath;
|
||||
}
|
||||
") = $nonCaPath ]]
|
||||
|
||||
[ -e $nonCaPath ]
|
||||
|
||||
# 'toPath' set to empty string should fail but print the expected path.
|
||||
nix eval -v --json --expr "
|
||||
builtins.fetchClosure {
|
||||
fromStore = \"file://$cacheDir\";
|
||||
fromPath = $nonCaPath;
|
||||
toPath = \"\";
|
||||
}
|
||||
" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath"
|
||||
|
||||
# If fromPath is CA, then toPath isn't needed.
|
||||
nix copy --to file://$cacheDir $caPath
|
||||
|
||||
[[ $(nix eval -v --raw --expr "
|
||||
builtins.fetchClosure {
|
||||
fromStore = \"file://$cacheDir\";
|
||||
fromPath = $caPath;
|
||||
}
|
||||
") = $caPath ]]
|
||||
|
|
@ -96,7 +96,8 @@ nix_tests = \
|
|||
describe-stores.sh \
|
||||
nix-profile.sh \
|
||||
suggestions.sh \
|
||||
store-ping.sh
|
||||
store-ping.sh \
|
||||
fetchClosure.sh
|
||||
|
||||
ifeq ($(HAVE_LIBCPUID), 1)
|
||||
nix_tests += compute-levels.sh
|
||||
|
|
|
|||
Loading…
Reference in a new issue