From 2e199673a523fa81de31ffdd2a25976ce0814631 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 19:43:53 +0100 Subject: [PATCH 1/6] Use `RealisedPath`s in `copyPaths` That way we can copy the realisations too (in addition to the store paths themselves) --- src/libstore/store-api.cc | 31 ++++++++++++++---------- src/libstore/store-api.hh | 9 +++---- src/nix-copy-closure/nix-copy-closure.cc | 6 ++--- src/nix/copy.cc | 13 +++++----- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2658f7617..529c34de5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -783,6 +783,24 @@ void copyStorePath(ref srcStore, ref dstStore, } +std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + StorePathSet storePaths; + std::set realisations; + for (auto path : paths) { + storePaths.insert(path.path()); + if (auto realisation = std::get_if(&path.raw)) + realisations.insert(*realisation); + } + auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + + return pathsMap; +} + std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { @@ -796,7 +814,6 @@ std::map copyPaths(ref srcStore, ref dstStor for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -871,21 +888,9 @@ std::map copyPaths(ref srcStore, ref dstStor nrDone++; showProgress(); }); - return pathsMap; } - -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, - SubstituteFlag substitute) -{ - StorePathSet closure; - srcStore->computeFSClosure(storePaths, closure); - copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); -} - - std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) { std::string path; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6dcd43ed1..63b26422a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,15 +752,12 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const StorePathSet & storePaths, + const RealisedPath::Set&, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); - - -/* Copy the closure of the specified paths from one store to another. */ -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet& paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 5e8cc515b..02ccbe541 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv) auto to = toMode ? openStore(remoteUri) : openStore(); auto from = toMode ? openStore() : openStore(remoteUri); - StorePathSet storePaths2; + RealisedPath::Set storePaths2; for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - StorePathSet closure; - from->computeFSClosure(storePaths2, closure, false, includeOutputs); + RealisedPath::Set closure; + RealisedPath::closure(*from, storePaths2, closure); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c56a1def1..f59f7c76b 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : StorePathsCommand +struct CmdCopy : RealisedPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; - using StorePathsCommand::run; + using RealisedPathsCommand::run; CmdCopy() - : StorePathsCommand(true) + : RealisedPathsCommand(true) { addFlag({ .longName = "from", @@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - StorePathsCommand::run(store); + RealisedPathsCommand::run(store); } - void run(ref srcStore, StorePaths storePaths) override + void run(ref srcStore, std::vector paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), + copyPaths( + srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), NoRepair, checkSigs, substitute); } }; From aead35531a0630b19e41348e103b2d105e2d8dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:37:05 +0100 Subject: [PATCH 2/6] Add a test for the copy of CA paths --- tests/local.mk | 1 + tests/nix-copy-content-addressed.sh | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 tests/nix-copy-content-addressed.sh diff --git a/tests/local.mk b/tests/local.mk index 06be8cec1..a504e397e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,6 +38,7 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ content-addressed.sh \ + nix-copy-content-addressed.sh \ build.sh \ compute-levels.sh # parallel.sh diff --git a/tests/nix-copy-content-addressed.sh b/tests/nix-copy-content-addressed.sh new file mode 100755 index 000000000..2e0dea2d2 --- /dev/null +++ b/tests/nix-copy-content-addressed.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +source common.sh + +# Globally enable the ca derivations experimental flag +sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +ensureCorrectlyCopied () { + attrPath="$1" + nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath" +} + +testOneCopy () { + clearStore + rm -rf "$REMOTE_STORE_DIR" + + attrPath="$1" + nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + + ensureCorrectlyCopied "$attrPath" + + # Ensure that we can copy back what we put in the store + clearStore + nix copy --from $REMOTE_STORE \ + --file ./content-addressed.nix "$attrPath" \ + --no-check-sigs +} + +for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do + testOneCopy "$attrPath" +done From f67ff1f5756018387a2d23c8f6772580192d30ad Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 17:58:28 +0100 Subject: [PATCH 3/6] Don't crash when copying realisations to a non-ca remote Rather throw a proper exception, and catch&log it on the client side --- src/libstore/globals.cc | 7 ++++++- src/libstore/globals.hh | 18 ++++++++++++++---- src/libstore/local-store.cc | 1 + src/libstore/store-api.cc | 14 ++++++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2780e0bf5..8d44003f4 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) return std::find(f.begin(), f.end(), name) != f.end(); } +MissingExperimentalFeature::MissingExperimentalFeature(std::string feature) + : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature) + , missingFeature(feature) + {} + void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + throw MissingExperimentalFeature(name); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index df61d6417..25351f55c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,6 +45,16 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +/* MakeError(MissingExperimentalFeature, Error); */ +class MissingExperimentalFeature: public Error +{ +public: + std::string missingFeature; + + MissingExperimentalFeature(std::string feature); + virtual const char* sname() const override { return "MissingExperimentalFeature"; } +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -632,7 +642,7 @@ public: is `root`. > **Warning** - > + > > Adding a user to `trusted-users` is essentially equivalent to > giving that user root access to the system. For example, the user > can set `sandbox-paths` and thereby obtain read access to @@ -722,13 +732,13 @@ public: The program executes with no arguments. The program's environment contains the following environment variables: - - `DRV_PATH` + - `DRV_PATH` The derivation for the built paths. Example: `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - `OUT_PATHS` + - `OUT_PATHS` Output paths of the built derivation, separated by a space character. @@ -759,7 +769,7 @@ public: documentation](https://ec.haxx.se/usingcurl-netrc.html). > **Note** - > + > > This must be an absolute path, and `~` is not resolved. For > example, `~/.netrc` won't resolve to your home directory's > `.netrc`. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0962418dd..90fb4a4bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { + settings.requireExperimentalFeature("ca-derivations"); auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 529c34de5..ac1d8ee2c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -794,8 +794,18 @@ std::map copyPaths(ref srcStore, ref dstStor realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); - for (auto& realisation : realisations) { - dstStore->registerDrvOutput(realisation); + try { + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + } catch (MissingExperimentalFeature & e) { + // Don't fail if the remote doesn't support CA derivations is it might + // not be whithin our control to change that, and we might still want + // to at least copy the output paths. + if (e.missingFeature == "ca-derivations") + ignoreException(); + else + throw; } return pathsMap; From 3b76f8f252c12fbeb49aa2f6f695b4622e9fcc5d Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 18:02:26 +0100 Subject: [PATCH 4/6] Ensure that the ca-derivations bit is set when copying realisations This should already hold, but better ensure it for future-proof-nees --- src/libstore/store-api.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ac1d8ee2c..db84ec7a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,6 +786,7 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { + settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto path : paths) { From c182aac98ab6548c16b6686638591ba5b034026a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 25 Feb 2021 17:10:45 +0100 Subject: [PATCH 5/6] Apply @edolstra stylistic suggestions Mostly removing useless comments and adding spaces before `&` Co-authored-by: Eelco Dolstra --- src/libstore/globals.hh | 1 - src/libstore/store-api.cc | 6 +++--- src/libstore/store-api.hh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 25351f55c..a51d9c2f1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,7 +45,6 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -/* MakeError(MissingExperimentalFeature, Error); */ class MissingExperimentalFeature: public Error { public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index db84ec7a2..b7a3f7b11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -789,19 +789,19 @@ std::map copyPaths(ref srcStore, ref dstStor settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; - for (auto path : paths) { + for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - for (auto& realisation : realisations) { + for (auto & realisation : realisations) { dstStore->registerDrvOutput(realisation); } } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might - // not be whithin our control to change that, and we might still want + // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == "ca-derivations") ignoreException(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 63b26422a..742cd18db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,7 +752,7 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const RealisedPath::Set&, + const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From c43f446f4e3a1a8d91560b6ebbcc7d4fbbbf71c4 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 25 Feb 2021 16:58:27 +0100 Subject: [PATCH 6/6] Make `nix copy` work without the ca-derivations flag The experimental feature was by mistake required for `nix copy` to work at oll --- src/libstore/store-api.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b7a3f7b11..77c310988 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,13 +786,14 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto & path : paths) { storePaths.insert(path.path()); - if (auto realisation = std::get_if(&path.raw)) + if (auto realisation = std::get_if(&path.raw)) { + settings.requireExperimentalFeature("ca-derivations"); realisations.insert(*realisation); + } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try {