diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 8e60b2ccc..56fd53ed5 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "make-content-addressed.hh" namespace nix { @@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args std::optional fromStoreUrl; std::optional fromPath; + bool toCA = false; + std::optional toPath; for (auto & attr : *args[0]->attrs) { if (attr.name == "fromPath") { @@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); } + else if (attr.name == "toPath") { + state.forceValue(*attr.value, *attr.pos); + toCA = true; + if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + PathSet context; + toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + } + else if (attr.name == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); @@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args // FIXME: only allow some "trusted" store types (like BinaryCacheStore). auto fromStore = openStore(*fromStoreUrl); - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + if (toCA) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } else { + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + toPath = fromPath; + } /* In pure mode, require a CA path. */ if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*fromPath); + auto info = state.store->queryPathInfo(*toPath); if (!info->isContentAddressed(*state.store)) throw Error({ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*fromPath)), + state.store->printStorePath(*toPath)), .errPos = pos }); } - auto fromPathS = state.store->printStorePath(*fromPath); - v.mkString(fromPathS, {fromPathS}); + auto toPathS = state.store->printStorePath(*toPath); + v.mkString(toPathS, {toPathS}); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc new file mode 100644 index 000000000..0b95ff37c --- /dev/null +++ b/src/libstore/make-content-addressed.cc @@ -0,0 +1,79 @@ +#include "make-content-addressed.hh" +#include "references.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths) +{ + // FIXME: use closure of storePaths. + + auto paths = srcStore.topoSortPaths(storePaths); + + std::reverse(paths.begin(), paths.end()); + + std::map remappings; + + for (auto & path : paths) { + auto pathS = srcStore.printStorePath(path); + auto oldInfo = srcStore.queryPathInfo(path); + std::string oldHashPart(path.hashPart()); + + StringSink sink; + srcStore.narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + sink.s = rewriteStrings(sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink(sink.s); + + auto narHash = hashModuloSink.finish().first; + + ValidPathInfo info { + dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path); + info.narSize = sink.s.size(); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + }; + + printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path)); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); + rsink2(sink.s); + rsink2.flush(); + }); + + dstStore.addToStore(info, *source); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + + return remappings; +} + +} diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh new file mode 100644 index 000000000..c4a66ed41 --- /dev/null +++ b/src/libstore/make-content-addressed.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +std::map makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths); + +} diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 2e75a3b61..a8579ea7c 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "store-api.hh" -#include "references.hh" +#include "make-content-addressed.hh" #include "common-args.hh" #include "json.hh" @@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON void run(ref store, StorePaths && storePaths) override { - auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); + auto remappings = makeContentAddressed(*store, *store, + StorePathSet(storePaths.begin(), storePaths.end())); - std::reverse(paths.begin(), paths.end()); - - std::map remappings; - - auto jsonRoot = json ? std::make_unique(std::cout) : nullptr; - auto jsonRewrites = json ? std::make_unique(jsonRoot->object("rewrites")) : nullptr; - - for (auto & path : paths) { - auto pathS = store->printStorePath(path); - auto oldInfo = store->queryPathInfo(path); - std::string oldHashPart(path.hashPart()); - - StringSink sink; - store->narFromPath(path, sink); - - StringMap rewrites; - - StorePathSet references; - bool hasSelfReference = false; - for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + if (json) { + JSONObject jsonRoot(std::cout); + JSONObject jsonRewrites(jsonRoot.object("rewrites")); + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second)); + } + } else { + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + notice("rewrote '%s' to '%s'", + store->printStorePath(path), + store->printStorePath(i->second)); } - - sink.s = rewriteStrings(sink.s, rewrites); - - HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink(sink.s); - - auto narHash = hashModuloSink.finish().first; - - ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); - info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; - - if (!json) - notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - store->addToStore(info, *source); - - if (json) - jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); - - remappings.insert_or_assign(std::move(path), std::move(info.path)); } } };