Merge pull request #9032 from Ma27/structured-attrs-env-vars

structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE

(cherry picked from commit 3c042f3b0b0a7ef9c47bf049f5410dbd4aac9e90)
Change-Id: I7e41838338ee1edf31fff6f9e354c3db2bba6c0e

doc/manual/src/language/advanced-attributes.md

@@ -274,18 +274,21 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
-    attributes are serialised in JSON format and made available to the
+    attributes are serialised into a file in JSON format. The environment variable
-    builder via the file `.attrs.json` in the builder’s temporary
+    `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
-    directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files
+    and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
-    have no size restrictions, unlike process environments.
+    [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
+    unlike process environments.
 
     It also makes it possible to tweak derivation settings in a structured way; see
     [`outputChecks`](#adv-attr-outputChecks) for example.
 
     As a convenience to Bash builders,
-    Nix writes a script named `.attrs.sh` to the builder’s directory
+    Nix writes a script that initialises shell variables
-    that initialises shell variables corresponding to all attributes
+    corresponding to all attributes that are representable in Bash. The
-    that are representable in Bash. This includes non-nested
+    environment variable `NIX_ATTRS_SH_FILE` points to the exact
+    location of the script, both in a build and a
+    [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
     (associative) arrays. For example, the attribute `hardening.format = true`
     ends up as the Bash associative array element `${hardening[format]}`.
 

doc/manual/src/language/derivations.md

@@ -123,6 +123,11 @@ The builder is executed as follows:
     
       - `NIX_STORE` is set to the path of the top-level Nix store
         directory (typically, `/nix/store`).
+
+      - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
+        is set to `true` for the dervation. A detailed explanation of this
+        behavior can be found in the
+        [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
     
       - For each output declared in `outputs`, the corresponding
         environment variable is set to point to the intended path in the

src/nix/develop.cc

@@ -8,9 +8,12 @@
 #include "derivations.hh"
 #include "progress-bar.hh"
 #include "run.hh"
+#include "util.hh"
 
+#include <iterator>
 #include <memory>
 #include <nlohmann/json.hpp>
+#include <algorithm>
 
 using namespace nix;
 
@@ -51,6 +54,7 @@ struct BuildEnvironment
 
     std::map<std::string, Value> vars;
     std::map<std::string, std::string> bashFunctions;
+    std::optional<std::pair<std::string, std::string>> structuredAttrs;
 
     static BuildEnvironment fromJSON(std::string_view in)
     {
@@ -74,6 +78,10 @@ struct BuildEnvironment
             res.bashFunctions.insert({name, def});
         }
 
+        if (json.contains("structuredAttrs")) {
+            res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]};
+        }
+
         return res;
     }
 
@@ -102,6 +110,13 @@ struct BuildEnvironment
 
         res["bashFunctions"] = bashFunctions;
 
+        if (providesStructuredAttrs()) {
+            auto contents = nlohmann::json::object();
+            contents[".attrs.sh"] = getAttrsSH();
+            contents[".attrs.json"] = getAttrsJSON();
+            res["structuredAttrs"] = std::move(contents);
+        }
+
         auto json = res.dump();
 
         assert(BuildEnvironment::fromJSON(json) == *this);
@@ -109,6 +124,23 @@ struct BuildEnvironment
         return json;
     }
 
+    bool providesStructuredAttrs() const
+    {
+        return structuredAttrs.has_value();
+    }
+
+    std::string getAttrsJSON() const
+    {
+        assert(providesStructuredAttrs());
+        return structuredAttrs->first;
+    }
+
+    std::string getAttrsSH() const
+    {
+        assert(providesStructuredAttrs());
+        return structuredAttrs->second;
+    }
+
     void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
     {
         for (auto & [name, value] : vars) {
@@ -290,6 +322,7 @@ struct Common : InstallableCommand, MixProfile
     std::string makeRcScript(
         ref<Store> store,
         const BuildEnvironment & buildEnvironment,
+        const Path & tmpDir,
         const Path & outputsDir = absPath(".") + "/outputs")
     {
         // A list of colon-separated environment variables that should be
@@ -352,9 +385,48 @@ struct Common : InstallableCommand, MixProfile
             }
         }
 
+        if (buildEnvironment.providesStructuredAttrs()) {
+            fixupStructuredAttrs(
+                "sh",
+                "NIX_ATTRS_SH_FILE",
+                buildEnvironment.getAttrsSH(),
+                rewrites,
+                buildEnvironment,
+                tmpDir
+            );
+            fixupStructuredAttrs(
+                "json",
+                "NIX_ATTRS_JSON_FILE",
+                buildEnvironment.getAttrsJSON(),
+                rewrites,
+                buildEnvironment,
+                tmpDir
+            );
+        }
+
         return rewriteStrings(script, rewrites);
     }
 
+    /**
+     * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file
+     * that's accessible from the interactive shell session.
+     */
+    void fixupStructuredAttrs(
+        const std::string & ext,
+        const std::string & envVar,
+        const std::string & content,
+        StringMap & rewrites,
+        const BuildEnvironment & buildEnvironment,
+        const Path & tmpDir)
+    {
+        auto targetFilePath = tmpDir + "/.attrs." + ext;
+        writeFile(targetFilePath, content);
+
+        auto fileInBuilderEnv = buildEnvironment.vars.find(envVar);
+        assert(fileInBuilderEnv != buildEnvironment.vars.end());
+        rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath});
+    }
+
     Strings getDefaultFlakeAttrPaths() override
     {
         Strings paths{
@@ -486,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
-        auto script = makeRcScript(store, buildEnvironment);
+        AutoDelete tmpDir(createTempDir("", "nix-develop"), true);
+
+        auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir);
 
         if (verbosity >= lvlDebug)
             script += "set -x\n";
@@ -618,10 +692,12 @@ struct CmdPrintDevEnv : Common, MixJSON
 
         stopProgressBar();
 
-        logger->writeToStdout(
+        if (json) {
-            json
+            logger->writeToStdout(buildEnvironment.toJSON());
-            ? buildEnvironment.toJSON()
+        } else {
-            : makeRcScript(store, buildEnvironment));
+            AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true);
+            logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir));
+        }
     }
 };
 

src/nix/get-env.sh

@@ -1,5 +1,5 @@
 set -e
-if [ -e .attrs.sh ]; then source .attrs.sh; fi
+if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi
 
 export IN_NIX_SHELL=impure
 export dontAddDisableDepTrack=1
@@ -101,7 +101,21 @@ __dumpEnv() {
 
         printf "}"
     done < <(printf "%s\n" "$__vars")
-    printf '\n  }\n}'
+    printf '\n  }'
+
+    if [ -e "$NIX_ATTRS_SH_FILE" ]; then
+        printf ',\n  "structuredAttrs": {\n    '
+        __escapeString ".attrs.sh"
+        printf ': '
+        __escapeString "$(<"$NIX_ATTRS_SH_FILE")"
+        printf ',\n    '
+        __escapeString ".attrs.json"
+        printf ': '
+        __escapeString "$(<"$NIX_ATTRS_JSON_FILE")"
+        printf '\n  }'
+    fi
+
+    printf '\n}'
 }
 
 __escapeString() {
@@ -117,7 +131,7 @@ __escapeString() {
 # In case of `__structuredAttrs = true;` the list of outputs is an associative
 # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist`
 # must contain the array's keys (hence `${!...[@]}`) in this case.
-if [ -e .attrs.sh ]; then
+if [ -e "$NIX_ATTRS_SH_FILE" ]; then
     __olist="${!outputs[@]}"
 else
     __olist=$outputs

tests/functional/build-hook-ca-fixed.nix

@@ -8,7 +8,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
     } // removeAttrs args ["builder" "meta" "passthru"])

tests/functional/build-hook.nix

@@ -14,7 +14,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
     } // removeAttrs args ["builder" "meta" "passthru"]
     // caArgs)
     // { meta = args.meta or {}; passthru = args.passthru or {}; };

tests/functional/config.nix.in

@@ -20,7 +20,10 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
       PATH = path;
     } // caArgs // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };

tests/functional/failing.nix

@@ -6,7 +6,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 in

tests/functional/hermetic.nix

@@ -14,7 +14,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
     } // removeAttrs args ["builder" "meta" "passthru"]
     // caArgs)
     // { meta = args.meta or {}; passthru = args.passthru or {}; };

tests/functional/structured-attrs.sh

@@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
 
 export NIX_BUILD_SHELL=$SHELL
 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
-    --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
+    --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
+
+nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
 
 # `nix develop` is a slightly special way of dealing with environment vars, it parses
 # these from a shell-file exported from a derivation. This is to test especially `outputs`
 # (which is an associative array in thsi case) being fine.
 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"'
+
+nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE='
+nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE='
+nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE'
+
+jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)"
+
+test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")"
+
+test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')"