nrabulinski/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

fetchClosure: Allow a path to be rewritten to CA on the fly

Browse source 
The advantage is that the resulting closure doesn't need to be signed,
so you don't need to configure any binary cache keys on the client.
This commit is contained in:
Eelco Dolstra 2022-03-22 21:14:58 +01:00
parent 7f6fe8ca1d
commit 545c2d0d8c
4 changed files with 150 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
src/libexpr/primops/fetchClosure.cc
View file

@ -1,5 +1,6 @@
#include "primops.hh"
#include "store-api.hh"
#include "make-content-addressed.hh"
namespace nix {
@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
std::optional<std::string> fromStoreUrl;
std::optional<StorePath> fromPath;
bool toCA = false;
std::optional<StorePath> toPath;
for (auto & attr : *args[0]->attrs) {
if (attr.name == "fromPath") {
@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
}
else if (attr.name == "toPath") {
state.forceValue(*attr.value, *attr.pos);
toCA = true;
if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
PathSet context;
toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
}
}
else if (attr.name == "fromStore")
fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
// FIXME: only allow some "trusted" store types (like BinaryCacheStore).
auto fromStore = openStore(*fromStoreUrl);
copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
if (toCA) {
auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
auto i = remappings.find(*fromPath);
assert(i != remappings.end());
if (toPath && *toPath != i->second)
throw Error({
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second),
state.store->printStorePath(*toPath)),
.errPos = pos
});
if (!toPath)
throw Error({
.msg = hintfmt(
"rewriting '%s' to content-addressed form yielded '%s'; "
"please set this in the 'toPath' attribute passed to 'fetchClosure'",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second)),
.errPos = pos
});
} else {
copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
toPath = fromPath;
}
/* In pure mode, require a CA path. */
if (evalSettings.pureEval) {
auto info = state.store->queryPathInfo(*fromPath);
auto info = state.store->queryPathInfo(*toPath);
if (!info->isContentAddressed(*state.store))
throw Error({
.msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
state.store->printStorePath(*fromPath)),
state.store->printStorePath(*toPath)),
.errPos = pos
});
}
auto fromPathS = state.store->printStorePath(*fromPath);
v.mkString(fromPathS, {fromPathS});
auto toPathS = state.store->printStorePath(*toPath);
v.mkString(toPathS, {toPathS});
}
static RegisterPrimOp primop_fetchClosure({

79
src/libstore/make-content-addressed.cc Normal file
View file

@ -0,0 +1,79 @@
#include "make-content-addressed.hh"
#include "references.hh"
namespace nix {
std::map<StorePath, StorePath> makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths)
{
// FIXME: use closure of storePaths.
auto paths = srcStore.topoSortPaths(storePaths);
std::reverse(paths.begin(), paths.end());
std::map<StorePath, StorePath> remappings;
for (auto & path : paths) {
auto pathS = srcStore.printStorePath(path);
auto oldInfo = srcStore.queryPathInfo(path);
std::string oldHashPart(path.hashPart());
StringSink sink;
srcStore.narFromPath(path, sink);
StringMap rewrites;
StorePathSet references;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
references.insert(std::move(replacement));
}
}
sink.s = rewriteStrings(sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink(sink.s);
auto narHash = hashModuloSink.finish().first;
ValidPathInfo info {
dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
narHash,
};
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path));
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
rsink2(sink.s);
rsink2.flush();
});
dstStore.addToStore(info, *source);
remappings.insert_or_assign(std::move(path), std::move(info.path));
}
return remappings;
}
}

12
src/libstore/make-content-addressed.hh Normal file
View file

@ -0,0 +1,12 @@
#pragma once
#include "store-api.hh"
namespace nix {
std::map<StorePath, StorePath> makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths);
}

85
src/nix/make-content-addressable.cc
View file

@ -1,6 +1,6 @@
#include "command.hh"
#include "store-api.hh"
#include "references.hh"
#include "make-content-addressed.hh"
#include "common-args.hh"
#include "json.hh"
@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
void run(ref<Store> store, StorePaths && storePaths) override
{
auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
auto remappings = makeContentAddressed(*store, *store,
StorePathSet(storePaths.begin(), storePaths.end()));
std::reverse(paths.begin(), paths.end());
std::map<StorePath, StorePath> remappings;
auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
for (auto & path : paths) {
auto pathS = store->printStorePath(path);
auto oldInfo = store->queryPathInfo(path);
std::string oldHashPart(path.hashPart());
StringSink sink;
store->narFromPath(path, sink);
StringMap rewrites;
StorePathSet references;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
references.insert(std::move(replacement));
}
if (json) {
JSONObject jsonRoot(std::cout);
JSONObject jsonRewrites(jsonRoot.object("rewrites"));
for (auto & path : storePaths) {
auto i = remappings.find(path);
assert(i != remappings.end());
jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second));
}
} else {
for (auto & path : storePaths) {
auto i = remappings.find(path);
assert(i != remappings.end());
notice("rewrote '%s' to '%s'",
store->printStorePath(path),
store->printStorePath(i->second));
}
sink.s = rewriteStrings(sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink(sink.s);
auto narHash = hashModuloSink.finish().first;
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
narHash,
};
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
if (!json)
notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
rsink2(sink.s);
rsink2.flush();
});
store->addToStore(info, *source);
if (json)
jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
remappings.insert_or_assign(std::move(path), std::move(info.path));
}
}
};