From 545c2d0d8cbac86c169a6dd049c1ed9c3913774d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Mar 2022 21:14:58 +0100 Subject: [PATCH] fetchClosure: Allow a path to be rewritten to CA on the fly The advantage is that the resulting closure doesn't need to be signed, so you don't need to configure any binary cache keys on the client. --- src/libexpr/primops/fetchClosure.cc | 46 ++++++++++++-- src/libstore/make-content-addressed.cc | 79 ++++++++++++++++++++++++ src/libstore/make-content-addressed.hh | 12 ++++ src/nix/make-content-addressable.cc | 85 ++++++-------------------- 4 files changed, 150 insertions(+), 72 deletions(-) create mode 100644 src/libstore/make-content-addressed.cc create mode 100644 src/libstore/make-content-addressed.hh diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 8e60b2ccc..56fd53ed5 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "make-content-addressed.hh" namespace nix { @@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args std::optional fromStoreUrl; std::optional fromPath; + bool toCA = false; + std::optional toPath; for (auto & attr : *args[0]->attrs) { if (attr.name == "fromPath") { @@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); } + else if (attr.name == "toPath") { + state.forceValue(*attr.value, *attr.pos); + toCA = true; + if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + PathSet context; + toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + } + else if (attr.name == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); @@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args // FIXME: only allow some "trusted" store types (like BinaryCacheStore). auto fromStore = openStore(*fromStoreUrl); - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + if (toCA) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } else { + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + toPath = fromPath; + } /* In pure mode, require a CA path. */ if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*fromPath); + auto info = state.store->queryPathInfo(*toPath); if (!info->isContentAddressed(*state.store)) throw Error({ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*fromPath)), + state.store->printStorePath(*toPath)), .errPos = pos }); } - auto fromPathS = state.store->printStorePath(*fromPath); - v.mkString(fromPathS, {fromPathS}); + auto toPathS = state.store->printStorePath(*toPath); + v.mkString(toPathS, {toPathS}); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc new file mode 100644 index 000000000..0b95ff37c --- /dev/null +++ b/src/libstore/make-content-addressed.cc @@ -0,0 +1,79 @@ +#include "make-content-addressed.hh" +#include "references.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths) +{ + // FIXME: use closure of storePaths. + + auto paths = srcStore.topoSortPaths(storePaths); + + std::reverse(paths.begin(), paths.end()); + + std::map remappings; + + for (auto & path : paths) { + auto pathS = srcStore.printStorePath(path); + auto oldInfo = srcStore.queryPathInfo(path); + std::string oldHashPart(path.hashPart()); + + StringSink sink; + srcStore.narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + sink.s = rewriteStrings(sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink(sink.s); + + auto narHash = hashModuloSink.finish().first; + + ValidPathInfo info { + dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path); + info.narSize = sink.s.size(); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + }; + + printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path)); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); + rsink2(sink.s); + rsink2.flush(); + }); + + dstStore.addToStore(info, *source); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + + return remappings; +} + +} diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh new file mode 100644 index 000000000..c4a66ed41 --- /dev/null +++ b/src/libstore/make-content-addressed.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths); + +} diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 2e75a3b61..a8579ea7c 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "store-api.hh" -#include "references.hh" +#include "make-content-addressed.hh" #include "common-args.hh" #include "json.hh" @@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON void run(ref store, StorePaths && storePaths) override { - auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); + auto remappings = makeContentAddressed(*store, *store, + StorePathSet(storePaths.begin(), storePaths.end())); - std::reverse(paths.begin(), paths.end()); - - std::map remappings; - - auto jsonRoot = json ? std::make_unique(std::cout) : nullptr; - auto jsonRewrites = json ? std::make_unique(jsonRoot->object("rewrites")) : nullptr; - - for (auto & path : paths) { - auto pathS = store->printStorePath(path); - auto oldInfo = store->queryPathInfo(path); - std::string oldHashPart(path.hashPart()); - - StringSink sink; - store->narFromPath(path, sink); - - StringMap rewrites; - - StorePathSet references; - bool hasSelfReference = false; - for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + if (json) { + JSONObject jsonRoot(std::cout); + JSONObject jsonRewrites(jsonRoot.object("rewrites")); + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second)); + } + } else { + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + notice("rewrote '%s' to '%s'", + store->printStorePath(path), + store->printStorePath(i->second)); } - - sink.s = rewriteStrings(sink.s, rewrites); - - HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink(sink.s); - - auto narHash = hashModuloSink.finish().first; - - ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); - info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; - - if (!json) - notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - store->addToStore(info, *source); - - if (json) - jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); - - remappings.insert_or_assign(std::move(path), std::move(info.path)); } } };