Merge pull request #9032 from Ma27/structured-attrs-env-vars

structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE

(cherry picked from commit 3c042f3b0b0a7ef9c47bf049f5410dbd4aac9e90)
Change-Id: I7e41838338ee1edf31fff6f9e354c3db2bba6c0e
This commit is contained in:
eldritch horrors 2024-03-07 10:46:47 +01:00
parent ca03f7cc28
commit 9eb58f5209
10 changed files with 146 additions and 21 deletions

View file

@ -274,18 +274,21 @@ Derivations can declare some infrequently used optional attributes.
- [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
If the special attribute `__structuredAttrs` is set to `true`, the other derivation If the special attribute `__structuredAttrs` is set to `true`, the other derivation
attributes are serialised in JSON format and made available to the attributes are serialised into a file in JSON format. The environment variable
builder via the file `.attrs.json` in the builders temporary `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
have no size restrictions, unlike process environments. [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
unlike process environments.
It also makes it possible to tweak derivation settings in a structured way; see It also makes it possible to tweak derivation settings in a structured way; see
[`outputChecks`](#adv-attr-outputChecks) for example. [`outputChecks`](#adv-attr-outputChecks) for example.
As a convenience to Bash builders, As a convenience to Bash builders,
Nix writes a script named `.attrs.sh` to the builders directory Nix writes a script that initialises shell variables
that initialises shell variables corresponding to all attributes corresponding to all attributes that are representable in Bash. The
that are representable in Bash. This includes non-nested environment variable `NIX_ATTRS_SH_FILE` points to the exact
location of the script, both in a build and a
[`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
(associative) arrays. For example, the attribute `hardening.format = true` (associative) arrays. For example, the attribute `hardening.format = true`
ends up as the Bash associative array element `${hardening[format]}`. ends up as the Bash associative array element `${hardening[format]}`.

View file

@ -123,6 +123,11 @@ The builder is executed as follows:
- `NIX_STORE` is set to the path of the top-level Nix store - `NIX_STORE` is set to the path of the top-level Nix store
directory (typically, `/nix/store`). directory (typically, `/nix/store`).
- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
is set to `true` for the dervation. A detailed explanation of this
behavior can be found in the
[section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
- For each output declared in `outputs`, the corresponding - For each output declared in `outputs`, the corresponding
environment variable is set to point to the intended path in the environment variable is set to point to the intended path in the

View file

@ -8,9 +8,12 @@
#include "derivations.hh" #include "derivations.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "run.hh" #include "run.hh"
#include "util.hh"
#include <iterator>
#include <memory> #include <memory>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <algorithm>
using namespace nix; using namespace nix;
@ -51,6 +54,7 @@ struct BuildEnvironment
std::map<std::string, Value> vars; std::map<std::string, Value> vars;
std::map<std::string, std::string> bashFunctions; std::map<std::string, std::string> bashFunctions;
std::optional<std::pair<std::string, std::string>> structuredAttrs;
static BuildEnvironment fromJSON(std::string_view in) static BuildEnvironment fromJSON(std::string_view in)
{ {
@ -74,6 +78,10 @@ struct BuildEnvironment
res.bashFunctions.insert({name, def}); res.bashFunctions.insert({name, def});
} }
if (json.contains("structuredAttrs")) {
res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]};
}
return res; return res;
} }
@ -102,6 +110,13 @@ struct BuildEnvironment
res["bashFunctions"] = bashFunctions; res["bashFunctions"] = bashFunctions;
if (providesStructuredAttrs()) {
auto contents = nlohmann::json::object();
contents[".attrs.sh"] = getAttrsSH();
contents[".attrs.json"] = getAttrsJSON();
res["structuredAttrs"] = std::move(contents);
}
auto json = res.dump(); auto json = res.dump();
assert(BuildEnvironment::fromJSON(json) == *this); assert(BuildEnvironment::fromJSON(json) == *this);
@ -109,6 +124,23 @@ struct BuildEnvironment
return json; return json;
} }
bool providesStructuredAttrs() const
{
return structuredAttrs.has_value();
}
std::string getAttrsJSON() const
{
assert(providesStructuredAttrs());
return structuredAttrs->first;
}
std::string getAttrsSH() const
{
assert(providesStructuredAttrs());
return structuredAttrs->second;
}
void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
{ {
for (auto & [name, value] : vars) { for (auto & [name, value] : vars) {
@ -290,6 +322,7 @@ struct Common : InstallableCommand, MixProfile
std::string makeRcScript( std::string makeRcScript(
ref<Store> store, ref<Store> store,
const BuildEnvironment & buildEnvironment, const BuildEnvironment & buildEnvironment,
const Path & tmpDir,
const Path & outputsDir = absPath(".") + "/outputs") const Path & outputsDir = absPath(".") + "/outputs")
{ {
// A list of colon-separated environment variables that should be // A list of colon-separated environment variables that should be
@ -352,9 +385,48 @@ struct Common : InstallableCommand, MixProfile
} }
} }
if (buildEnvironment.providesStructuredAttrs()) {
fixupStructuredAttrs(
"sh",
"NIX_ATTRS_SH_FILE",
buildEnvironment.getAttrsSH(),
rewrites,
buildEnvironment,
tmpDir
);
fixupStructuredAttrs(
"json",
"NIX_ATTRS_JSON_FILE",
buildEnvironment.getAttrsJSON(),
rewrites,
buildEnvironment,
tmpDir
);
}
return rewriteStrings(script, rewrites); return rewriteStrings(script, rewrites);
} }
/**
* Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file
* that's accessible from the interactive shell session.
*/
void fixupStructuredAttrs(
const std::string & ext,
const std::string & envVar,
const std::string & content,
StringMap & rewrites,
const BuildEnvironment & buildEnvironment,
const Path & tmpDir)
{
auto targetFilePath = tmpDir + "/.attrs." + ext;
writeFile(targetFilePath, content);
auto fileInBuilderEnv = buildEnvironment.vars.find(envVar);
assert(fileInBuilderEnv != buildEnvironment.vars.end());
rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath});
}
Strings getDefaultFlakeAttrPaths() override Strings getDefaultFlakeAttrPaths() override
{ {
Strings paths{ Strings paths{
@ -486,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment
auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
auto script = makeRcScript(store, buildEnvironment); AutoDelete tmpDir(createTempDir("", "nix-develop"), true);
auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir);
if (verbosity >= lvlDebug) if (verbosity >= lvlDebug)
script += "set -x\n"; script += "set -x\n";
@ -618,10 +692,12 @@ struct CmdPrintDevEnv : Common, MixJSON
stopProgressBar(); stopProgressBar();
logger->writeToStdout( if (json) {
json logger->writeToStdout(buildEnvironment.toJSON());
? buildEnvironment.toJSON() } else {
: makeRcScript(store, buildEnvironment)); AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true);
logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir));
}
} }
}; };

View file

@ -1,5 +1,5 @@
set -e set -e
if [ -e .attrs.sh ]; then source .attrs.sh; fi if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi
export IN_NIX_SHELL=impure export IN_NIX_SHELL=impure
export dontAddDisableDepTrack=1 export dontAddDisableDepTrack=1
@ -101,7 +101,21 @@ __dumpEnv() {
printf "}" printf "}"
done < <(printf "%s\n" "$__vars") done < <(printf "%s\n" "$__vars")
printf '\n }\n}' printf '\n }'
if [ -e "$NIX_ATTRS_SH_FILE" ]; then
printf ',\n "structuredAttrs": {\n '
__escapeString ".attrs.sh"
printf ': '
__escapeString "$(<"$NIX_ATTRS_SH_FILE")"
printf ',\n '
__escapeString ".attrs.json"
printf ': '
__escapeString "$(<"$NIX_ATTRS_JSON_FILE")"
printf '\n }'
fi
printf '\n}'
} }
__escapeString() { __escapeString() {
@ -117,7 +131,7 @@ __escapeString() {
# In case of `__structuredAttrs = true;` the list of outputs is an associative # In case of `__structuredAttrs = true;` the list of outputs is an associative
# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist`
# must contain the array's keys (hence `${!...[@]}`) in this case. # must contain the array's keys (hence `${!...[@]}`) in this case.
if [ -e .attrs.sh ]; then if [ -e "$NIX_ATTRS_SH_FILE" ]; then
__olist="${!outputs[@]}" __olist="${!outputs[@]}"
else else
__olist=$outputs __olist=$outputs

View file

@ -8,7 +8,10 @@ let
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
outputHashMode = "recursive"; outputHashMode = "recursive";
outputHashAlgo = "sha256"; outputHashAlgo = "sha256";
} // removeAttrs args ["builder" "meta" "passthru"]) } // removeAttrs args ["builder" "meta" "passthru"])

View file

@ -14,7 +14,10 @@ let
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta" "passthru"] } // removeAttrs args ["builder" "meta" "passthru"]
// caArgs) // caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; }; // { meta = args.meta or {}; passthru = args.passthru or {}; };

View file

@ -20,7 +20,10 @@ rec {
derivation ({ derivation ({
inherit system; inherit system;
builder = shell; builder = shell;
args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
PATH = path; PATH = path;
} // caArgs // removeAttrs args ["builder" "meta"]) } // caArgs // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; }; // { meta = args.meta or {}; };

View file

@ -6,7 +6,10 @@ let
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta"]) } // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; }; // { meta = args.meta or {}; };
in in

View file

@ -14,7 +14,10 @@ let
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta" "passthru"] } // removeAttrs args ["builder" "meta" "passthru"]
// caArgs) // caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; }; // { meta = args.meta or {}; passthru = args.passthru or {}; };

View file

@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
export NIX_BUILD_SHELL=$SHELL export NIX_BUILD_SHELL=$SHELL
env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
--run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
# `nix develop` is a slightly special way of dealing with environment vars, it parses # `nix develop` is a slightly special way of dealing with environment vars, it parses
# these from a shell-file exported from a derivation. This is to test especially `outputs` # these from a shell-file exported from a derivation. This is to test especially `outputs`
# (which is an associative array in thsi case) being fine. # (which is an associative array in thsi case) being fine.
nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"'
nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE='
nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE='
nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE'
jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)"
test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")"
test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')"