lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Merge pull request #9032 from Ma27/structured-attrs-env-vars

Browse source 
structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE

(cherry picked from commit 3c042f3b0b0a7ef9c47bf049f5410dbd4aac9e90)
Change-Id: I7e41838338ee1edf31fff6f9e354c3db2bba6c0e
This commit is contained in:
eldritch horrors 2024-03-07 10:46:47 +01:00
parent ca03f7cc28
commit 9eb58f5209
10 changed files with 146 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
doc/manual/src/language/advanced-attributes.md
View file

@ -274,18 +274,21 @@ Derivations can declare some infrequently used optional attributes.
- [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
If the special attribute `__structuredAttrs` is set to `true`, the other derivation
attributes are serialised in JSON format and made available to the
builder via the file `.attrs.json` in the builder’s temporary
directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files
have no size restrictions, unlike process environments.
attributes are serialised into a file in JSON format. The environment variable
`NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
[`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
unlike process environments.
It also makes it possible to tweak derivation settings in a structured way; see
[`outputChecks`](#adv-attr-outputChecks) for example.
As a convenience to Bash builders,
Nix writes a script named `.attrs.sh` to the builder’s directory
that initialises shell variables corresponding to all attributes
that are representable in Bash. This includes non-nested
Nix writes a script that initialises shell variables
corresponding to all attributes that are representable in Bash. The
environment variable `NIX_ATTRS_SH_FILE` points to the exact
location of the script, both in a build and a
[`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
(associative) arrays. For example, the attribute `hardening.format = true`
ends up as the Bash associative array element `${hardening[format]}`.

5
doc/manual/src/language/derivations.md
View file

@ -123,6 +123,11 @@ The builder is executed as follows:
- `NIX_STORE` is set to the path of the top-level Nix store
directory (typically, `/nix/store`).
- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
is set to `true` for the dervation. A detailed explanation of this
behavior can be found in the
[section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
- For each output declared in `outputs`, the corresponding
environment variable is set to point to the intended path in the

86
src/nix/develop.cc
View file

@ -8,9 +8,12 @@
#include "derivations.hh"
#include "progress-bar.hh"
#include "run.hh"
#include "util.hh"
#include <iterator>
#include <memory>
#include <nlohmann/json.hpp>
#include <algorithm>
using namespace nix;
@ -51,6 +54,7 @@ struct BuildEnvironment
std::map<std::string, Value> vars;
std::map<std::string, std::string> bashFunctions;
std::optional<std::pair<std::string, std::string>> structuredAttrs;
static BuildEnvironment fromJSON(std::string_view in)
{
@ -74,6 +78,10 @@ struct BuildEnvironment
res.bashFunctions.insert({name, def});
}
if (json.contains("structuredAttrs")) {
res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]};
}
return res;
}
@ -102,6 +110,13 @@ struct BuildEnvironment
res["bashFunctions"] = bashFunctions;
if (providesStructuredAttrs()) {
auto contents = nlohmann::json::object();
contents[".attrs.sh"] = getAttrsSH();
contents[".attrs.json"] = getAttrsJSON();
res["structuredAttrs"] = std::move(contents);
}
auto json = res.dump();
assert(BuildEnvironment::fromJSON(json) == *this);
@ -109,6 +124,23 @@ struct BuildEnvironment
return json;
}
bool providesStructuredAttrs() const
{
return structuredAttrs.has_value();
}
std::string getAttrsJSON() const
{
assert(providesStructuredAttrs());
return structuredAttrs->first;
}
std::string getAttrsSH() const
{
assert(providesStructuredAttrs());
return structuredAttrs->second;
}
void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
{
for (auto & [name, value] : vars) {
@ -290,6 +322,7 @@ struct Common : InstallableCommand, MixProfile
std::string makeRcScript(
ref<Store> store,
const BuildEnvironment & buildEnvironment,
const Path & tmpDir,
const Path & outputsDir = absPath(".") + "/outputs")
{
// A list of colon-separated environment variables that should be
@ -352,9 +385,48 @@ struct Common : InstallableCommand, MixProfile
}
}
if (buildEnvironment.providesStructuredAttrs()) {
fixupStructuredAttrs(
"sh",
"NIX_ATTRS_SH_FILE",
buildEnvironment.getAttrsSH(),
rewrites,
buildEnvironment,
tmpDir
);
fixupStructuredAttrs(
"json",
"NIX_ATTRS_JSON_FILE",
buildEnvironment.getAttrsJSON(),
rewrites,
buildEnvironment,
tmpDir
);
}
return rewriteStrings(script, rewrites);
}
/**
* Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file
* that's accessible from the interactive shell session.
*/
void fixupStructuredAttrs(
const std::string & ext,
const std::string & envVar,
const std::string & content,
StringMap & rewrites,
const BuildEnvironment & buildEnvironment,
const Path & tmpDir)
{
auto targetFilePath = tmpDir + "/.attrs." + ext;
writeFile(targetFilePath, content);
auto fileInBuilderEnv = buildEnvironment.vars.find(envVar);
assert(fileInBuilderEnv != buildEnvironment.vars.end());
rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath});
}
Strings getDefaultFlakeAttrPaths() override
{
Strings paths{
@ -486,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment
auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
auto script = makeRcScript(store, buildEnvironment);
AutoDelete tmpDir(createTempDir("", "nix-develop"), true);
auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir);
if (verbosity >= lvlDebug)
script += "set -x\n";
@ -618,10 +692,12 @@ struct CmdPrintDevEnv : Common, MixJSON
stopProgressBar();
logger->writeToStdout(
json
? buildEnvironment.toJSON()
: makeRcScript(store, buildEnvironment));
if (json) {
logger->writeToStdout(buildEnvironment.toJSON());
} else {
AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true);
logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir));
}
}
};

20
src/nix/get-env.sh
View file

@ -1,5 +1,5 @@
set -e
if [ -e .attrs.sh ]; then source .attrs.sh; fi
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi
export IN_NIX_SHELL=impure
export dontAddDisableDepTrack=1
@ -101,7 +101,21 @@ __dumpEnv() {
printf "}"
done < <(printf "%s\n" "$__vars")
printf '\n }\n}'
printf '\n }'
if [ -e "$NIX_ATTRS_SH_FILE" ]; then
printf ',\n "structuredAttrs": {\n '
__escapeString ".attrs.sh"
printf ': '
__escapeString "$(<"$NIX_ATTRS_SH_FILE")"
printf ',\n '
__escapeString ".attrs.json"
printf ': '
__escapeString "$(<"$NIX_ATTRS_JSON_FILE")"
printf '\n }'
fi
printf '\n}'
}
__escapeString() {
@ -117,7 +131,7 @@ __escapeString() {
# In case of `__structuredAttrs = true;` the list of outputs is an associative
# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist`
# must contain the array's keys (hence `${!...[@]}`) in this case.
if [ -e .attrs.sh ]; then
if [ -e "$NIX_ATTRS_SH_FILE" ]; then
__olist="${!outputs[@]}"
else
__olist=$outputs

5
tests/functional/build-hook-ca-fixed.nix
View file

@ -8,7 +8,10 @@ let
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} // removeAttrs args ["builder" "meta" "passthru"])

5
tests/functional/build-hook.nix
View file

@ -14,7 +14,10 @@ let
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta" "passthru"]
// caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; };

5
tests/functional/config.nix.in
View file

@ -20,7 +20,10 @@ rec {
derivation ({
inherit system;
builder = shell;
args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
PATH = path;
} // caArgs // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };

5
tests/functional/failing.nix
View file

@ -6,7 +6,10 @@ let
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
in

5
tests/functional/hermetic.nix
View file

@ -14,7 +14,10 @@ let
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
eval "$buildCommand"
'')];
} // removeAttrs args ["builder" "meta" "passthru"]
// caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; };

14
tests/functional/structured-attrs.sh
View file

@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
export NIX_BUILD_SHELL=$SHELL
env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
--run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
--run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
# `nix develop` is a slightly special way of dealing with environment vars, it parses
# these from a shell-file exported from a derivation. This is to test especially `outputs`
# (which is an associative array in thsi case) being fine.
nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"'
nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE='
nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE='
nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE'
jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)"
test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")"
test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')"