From 9eb58f5209f9cd1fd18f33b691c6a613075dffc4 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 7 Mar 2024 10:46:47 +0100 Subject: [PATCH] Merge pull request #9032 from Ma27/structured-attrs-env-vars structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE (cherry picked from commit 3c042f3b0b0a7ef9c47bf049f5410dbd4aac9e90) Change-Id: I7e41838338ee1edf31fff6f9e354c3db2bba6c0e --- .../src/language/advanced-attributes.md | 17 ++-- doc/manual/src/language/derivations.md | 5 ++ src/nix/develop.cc | 86 +++++++++++++++++-- src/nix/get-env.sh | 20 ++++- tests/functional/build-hook-ca-fixed.nix | 5 +- tests/functional/build-hook.nix | 5 +- tests/functional/config.nix.in | 5 +- tests/functional/failing.nix | 5 +- tests/functional/hermetic.nix | 5 +- tests/functional/structured-attrs.sh | 14 ++- 10 files changed, 146 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 4907f7457..a5ad47a93 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -274,18 +274,21 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised in JSON format and made available to the - builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files - have no size restrictions, unlike process environments. + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see [`outputChecks`](#adv-attr-outputChecks) for example. As a convenience to Bash builders, - Nix writes a script named `.attrs.sh` to the builder’s directory - that initialises shell variables corresponding to all attributes - that are representable in Bash. This includes non-nested + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 79a09122a..e4a2fe714 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -123,6 +123,11 @@ The builder is executed as follows: - `NIX_STORE` is set to the path of the top-level Nix store directory (typically, `/nix/store`). + + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the dervation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). - For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 2790ff154..9ecfdef5d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,9 +8,12 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" +#include "util.hh" +#include #include #include +#include using namespace nix; @@ -51,6 +54,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -74,6 +78,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -102,6 +110,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -109,6 +124,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -290,6 +322,7 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, + const Path & tmpDir, const Path & outputsDir = absPath(".") + "/outputs") { // A list of colon-separated environment variables that should be @@ -352,9 +385,48 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + "sh", + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + "json", + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + const std::string & ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const Path & tmpDir) + { + auto targetFilePath = tmpDir + "/.attrs." + ext; + writeFile(targetFilePath, content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -486,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -618,10 +692,12 @@ struct CmdPrintDevEnv : Common, MixJSON stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b9..832cc2f11 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -117,7 +131,7 @@ __escapeString() { # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then +if [ -e "$NIX_ATTRS_SH_FILE" ]; then __olist="${!outputs[@]}" else __olist=$outputs diff --git a/tests/functional/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix index 4cb9e85d1..0ce6d9b12 100644 --- a/tests/functional/build-hook-ca-fixed.nix +++ b/tests/functional/build-hook-ca-fixed.nix @@ -8,7 +8,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; } // removeAttrs args ["builder" "meta" "passthru"]) diff --git a/tests/functional/build-hook.nix b/tests/functional/build-hook.nix index 7effd7903..99a13aee4 100644 --- a/tests/functional/build-hook.nix +++ b/tests/functional/build-hook.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/functional/config.nix.in b/tests/functional/config.nix.in index 7facbdcbc..00dc007e1 100644 --- a/tests/functional/config.nix.in +++ b/tests/functional/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/functional/failing.nix b/tests/functional/failing.nix index 2a0350d4d..d25e2d6b6 100644 --- a/tests/functional/failing.nix +++ b/tests/functional/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix index 4c9d7a51f..0513540f0 100644 --- a/tests/functional/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 378dbc735..f11992dcd 100644 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')"