From 15f7fa59bec1dbb4fa94bc1c2f9018b1f14f0d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 23 Sep 2022 23:30:29 +0200 Subject: [PATCH 1/3] `unsafeDiscardReferences` Adds a new boolean structured attribute `outputChecks..unsafeDiscardReferences` which disables scanning an output for runtime references. __structuredAttrs = true; outputChecks.out.unsafeDiscardReferences = true; This is useful when creating filesystem images containing their own embedded Nix store: they are self-contained blobs of data with no runtime dependencies. Setting this attribute requires the experimental feature `discard-references` to be enabled. --- .../src/language/advanced-attributes.md | 6 ++--- doc/manual/src/release-notes/rl-next.md | 12 +++++++++ src/libstore/build/local-derivation-goal.cc | 27 ++++++++++++++++--- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + tests/check-refs.nix | 9 +++++++ tests/check-refs.sh | 9 +++++++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..f02425b13 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,13 +207,13 @@ Derivations can declare some infrequently used optional attributes. the hash in either hexadecimal or base-32 notation. (See the [`nix-hash` command](../command-ref/nix-hash.md) for information about converting to and from base-32 notation.) - + - [`__contentAddressed`]{#adv-attr-__contentAddressed} If this **experimental** attribute is set to true, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - + This only has an effect if the `ca-derivations` experimental feature is enabled. + Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9f491efc8..51320d50a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -21,3 +21,15 @@ * Error traces have been reworked to provide detailed explanations and more accurate error locations. A short excerpt of the trace is now shown by default when an error occurs. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + outputChecks.out.unsafeDiscardReferences = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9d869d513..65340d6ce 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2319,11 +2319,30 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { + if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (!unsafeDiscardReferences->is_boolean()) + throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = unsafeDiscardReferences->get(); + } + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 9d90b0920..a5e40c9e5 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -67,4 +67,13 @@ rec { disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + outputChecks.out = { + unsafeDiscardReferences = true; + allowedReferences = []; + }; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 16bbabc40..65a72552a 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -40,3 +40,12 @@ nix-build -o $RESULT check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o $RESULT check-refs.nix -A test10 + +if isDaemonNewer 2.12pre20230103; then + enableFeatures discard-references + restartDaemon + + # test11 should succeed. + test11=$(nix-build -o $RESULT check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] +fi From 3c968191f1d54a172d2b245d080c67012f766f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 3 Jan 2023 18:53:01 +0100 Subject: [PATCH 2/3] move `unsafeDiscardReferences` out of `outputChecks` It's not a check. --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libstore/build/local-derivation-goal.cc | 14 ++++++-------- tests/check-refs.nix | 6 ++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 51320d50a..1e8a035b0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,7 +26,7 @@ to disable scanning a given output for runtime dependencies: ```nix __structuredAttrs = true; - outputChecks.out.unsafeDiscardReferences = true; + unsafeDiscardReferences.out = true; ``` This is useful e.g. when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 65340d6ce..d4ea68f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2321,14 +2321,12 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { - if (auto output = get(*outputChecks, outputName)) { - if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); - if (!unsafeDiscardReferences->is_boolean()) - throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); - discardReferences = unsafeDiscardReferences->get(); - } + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); } } } diff --git a/tests/check-refs.nix b/tests/check-refs.nix index a5e40c9e5..99d69a226 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -69,10 +69,8 @@ rec { test11 = makeTest 11 { __structuredAttrs = true; - outputChecks.out = { - unsafeDiscardReferences = true; - allowedReferences = []; - }; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; buildCommand = ''echo ${dep} > "''${outputs[out]}"''; }; From 7408776b003d8a7670c5d8eb393f22899a662b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:32:23 +0100 Subject: [PATCH 3/3] Fix the release-notes Slightly butchered during the merge --- doc/manual/src/release-notes/rl-next.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2f0f89cc6..7e8344e63 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,5 @@ +# Release X.Y (202?-??-??) + * A new function `builtins.readFileType` is available. It is similar to `builtins.readDir` but acts on a single file or directory. @@ -6,7 +8,7 @@ discovered by making multiple syscalls. This change makes these operations lazy such that these lookups will only be performed if the attribute is used. This optimization affects a minority of filesystems and operating systems. - + * In derivations that use structured attributes, you can now use `unsafeDiscardReferences` to disable scanning a given output for runtime dependencies: ```nix @@ -17,4 +19,4 @@ their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - This requires the `discard-references` experimental feature. \ No newline at end of file + This requires the `discard-references` experimental feature.