Merge branch 'master' into indexed-store-path-outputs

This commit is contained in:
John Ericson 2022-12-12 07:33:36 -05:00 committed by GitHub
commit 1879c7c95e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
83 changed files with 1547 additions and 1396 deletions

15
.github/CODEOWNERS vendored Normal file
View file

@ -0,0 +1,15 @@
# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers.
# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications.
#
# Merge permissions are required for maintaining an entry in this file.
# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/
# Default reviewers if nothing else matches
* @edolstra @thufschmitt
# This file
.github/CODEOWNERS @edolstra
# Public documentation
/doc @fricklerhandwerk
*.md @fricklerhandwerk

View file

@ -30,3 +30,7 @@ A clear and concise description of what you expected to happen.
**Additional context** **Additional context**
Add any other context about the problem here. Add any other context about the problem here.
**Priorities**
Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).

View file

@ -18,3 +18,7 @@ A clear and concise description of any alternative solutions or features you've
**Additional context** **Additional context**
Add any other context or screenshots about the feature request here. Add any other context or screenshots about the feature request here.
**Priorities**
Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).

36
.github/ISSUE_TEMPLATE/installer.md vendored Normal file
View file

@ -0,0 +1,36 @@
---
name: Installer issue
about: Report problems with installation
title: ''
labels: installer
assignees: ''
---
## Platform
<!-- select the platform on which you tried to install Nix -->
- [ ] Linux: <!-- state your distribution, e.g. Arch Linux, Ubuntu, ... -->
- [ ] macOS
- [ ] WSL
## Additional information
<!-- state special circumstances on your system or additional steps you have taken prior to installation -->
## Output
<details><summary>Output</summary>
```log
<!-- paste console output here and remove this comment -->
```
</details>
## Priorities
Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).

View file

@ -26,3 +26,6 @@ assignees: ''
<!-- propose a solution --> <!-- propose a solution -->
## Priorities
Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).

View file

@ -5,3 +5,7 @@ Please include relevant [release notes](https://github.com/NixOS/nix/blob/master
**Testing** **Testing**
If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).
**Priorities**
Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc).

View file

@ -21,7 +21,7 @@ jobs:
fetch-depth: 0 fetch-depth: 0
- name: Create backport PRs - name: Create backport PRs
# should be kept in sync with `version` # should be kept in sync with `version`
uses: zeebe-io/backport-action@v0.0.8 uses: zeebe-io/backport-action@v0.0.9
with: with:
# Config README: https://github.com/zeebe-io/backport-action#backport-action # Config README: https://github.com/zeebe-io/backport-action#backport-action
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}

View file

@ -1 +1 @@
2.12.0 2.13.0

View file

@ -1,16 +1,20 @@
with builtins; builtinsDump:
with import ./utils.nix; let
showBuiltin = name:
let
inherit (builtinsDump.${name}) doc args;
in
''
<dt id="builtins-${name}">
<a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a>
</dt>
<dd>
builtins: ${doc}
</dd>
'';
listArgs = args: builtins.concatStringsSep " " (map (s: "<var>${s}</var>") args);
in
with builtins; concatStringsSep "\n" (map showBuiltin (attrNames builtinsDump))
concatStrings (map
(name:
let builtin = builtins.${name}; in
"<dt id=\"builtins-${name}\"><a href=\"#builtins-${name}\"><code>${name} "
+ concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
+ "</code></a></dt>"
+ "<dd>\n\n"
+ builtin.doc
+ "\n\n</dd>"
)
(attrNames builtins))

View file

@ -99,6 +99,7 @@ let
in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
parsedToplevel = builtins.fromJSON toplevel; parsedToplevel = builtins.fromJSON toplevel;
manpages = processCommand { manpages = processCommand {
command = "nix"; command = "nix";
details = parsedToplevel; details = parsedToplevel;

View file

@ -1,29 +1,41 @@
with builtins; let
with import ./utils.nix; inherit (builtins) attrNames concatStringsSep isAttrs isBool;
inherit (import ./utils.nix) concatStrings squash splitLines;
in
options: optionsInfo:
let
showOption = name:
let
inherit (optionsInfo.${name}) description documentDefault defaultValue aliases;
result = squash ''
- <span id="conf-${name}">[`${name}`](#conf-${name})</span>
concatStrings (map ${indent " " body}
(name: '';
let option = options.${name}; in # separate body to cleanly handle indentation
" - [`${name}`](#conf-${name})" body = ''
+ "<p id=\"conf-${name}\"></p>\n\n" ${description}
+ concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n"
+ (if option.documentDefault **Default:** ${showDefault documentDefault defaultValue}
then " **Default:** " + (
if option.defaultValue == "" || option.defaultValue == [] ${showAliases aliases}
then "*empty*" '';
else if isBool option.defaultValue showDefault = documentDefault: defaultValue:
then (if option.defaultValue then "`true`" else "`false`") if documentDefault then
else # a StringMap value type is specified as a string, but
# n.b. a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in
# this shows the value type. The empty stringmap is "null" in # JSON, but that converts to `{ }` here.
# JSON, but that converts to "{ }" here. if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
(if isAttrs option.defaultValue then "`\"\"`" then "*empty*"
else "`" + toString option.defaultValue + "`")) + "\n\n" else if isBool defaultValue then
else " **Default:** *machine-specific*\n") if defaultValue then "`true`" else "`false`"
+ (if option.aliases != [] else "`${toString defaultValue}`"
then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" else "*machine-specific*";
else "") showAliases = aliases:
) if aliases == [] then "" else
(attrNames options)) "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
in concatStrings (map showOption (attrNames optionsInfo))

View file

@ -29,19 +29,19 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -
$(d)/%.1: $(d)/src/command-ref/%.md $(d)/%.1: $(d)/src/command-ref/%.md
@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
@cat $^ >> $^.tmp @cat $^ >> $^.tmp
$(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@ $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@
@rm $^.tmp @rm $^.tmp
$(d)/%.8: $(d)/src/command-ref/%.md $(d)/%.8: $(d)/src/command-ref/%.md
@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
@cat $^ >> $^.tmp @cat $^ >> $^.tmp
$(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@ $(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@
@rm $^.tmp @rm $^.tmp
$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
@cat $^ >> $^.tmp @cat $^ >> $^.tmp
$(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@ $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
@rm $^.tmp @rm $^.tmp
$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli

View file

@ -35,7 +35,6 @@ const redirects = {
"conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs", "conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs",
"conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size", "conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size",
"conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time", "conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time",
"conf-build-repeat": "command-ref/conf-file.html#conf-build-repeat",
"conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout", "conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout",
"conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot", "conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot",
"conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox", "conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox",
@ -47,7 +46,6 @@ const redirects = {
"conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout", "conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout",
"conf-cores": "command-ref/conf-file.html#conf-cores", "conf-cores": "command-ref/conf-file.html#conf-cores",
"conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook", "conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook",
"conf-enforce-determinism": "command-ref/conf-file.html#conf-enforce-determinism",
"conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations", "conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations",
"conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches", "conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches",
"conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms", "conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms",
@ -74,7 +72,6 @@ const redirects = {
"conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files", "conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files",
"conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook", "conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook",
"conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook", "conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook",
"conf-repeat": "command-ref/conf-file.html#conf-repeat",
"conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs", "conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs",
"conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval", "conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval",
"conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook", "conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook",

View file

@ -65,6 +65,7 @@
- [CLI guideline](contributing/cli-guideline.md) - [CLI guideline](contributing/cli-guideline.md)
- [Release Notes](release-notes/release-notes.md) - [Release Notes](release-notes/release-notes.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
- [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md)
- [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md)
- [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md)

View file

@ -121,37 +121,3 @@ error:
are not valid, so checking is not possible are not valid, so checking is not possible
Run the build without `--check`, and then try with `--check` again. Run the build without `--check`, and then try with `--check` again.
# Automatic and Optionally Enforced Determinism Verification
Automatically verify every build at build time by executing the build
multiple times.
Setting `repeat` and `enforce-determinism` in your `nix.conf` permits
the automated verification of every build Nix performs.
The following configuration will run each build three times, and will
require the build to be deterministic:
enforce-determinism = true
repeat = 2
Setting `enforce-determinism` to false as in the following
configuration will run the build multiple times, execute the build
hook, but will allow the build to succeed even if it does not build
reproducibly:
enforce-determinism = false
repeat = 1
An example output of this configuration:
```console
$ nix-build ./test.nix -A unstable
this derivation will be built:
/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
```

View file

@ -33,12 +33,17 @@ distribute the public key for verifying the authenticity of the paths.
example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
``` ```
Then, add the public key and the cache URL to your `nix.conf`'s Then update [`nix.conf`](../command-ref/conf-file.md) on any machine that will access the cache.
`trusted-public-keys` and `substituters` options: Add the cache URL to [`substituters`](../command-ref/conf-file.md#conf-substituters) and the public key to [`trusted-public-keys`](../command-ref/conf-file.md#conf-trusted-public-keys):
substituters = https://cache.nixos.org/ s3://example-nix-cache substituters = https://cache.nixos.org/ s3://example-nix-cache
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
Machines that build for the cache must sign derivations using the private key.
On those machines, add the path to the key file to the [`secret-key-files`](../command-ref/conf-file.md#conf-secret-key-files) field in their [`nix.conf`](../command-ref/conf-file.md):
secret-key-files = /etc/nix/key.private
We will restart the Nix daemon in a later step. We will restart the Nix daemon in a later step.
# Implementing the build hook # Implementing the build hook
@ -52,14 +57,12 @@ set -eu
set -f # disable globbing set -f # disable globbing
export IFS=' ' export IFS=' '
echo "Signing paths" $OUT_PATHS
nix store sign --key-file /etc/nix/key.private $OUT_PATHS
echo "Uploading paths" $OUT_PATHS echo "Uploading paths" $OUT_PATHS
exec nix copy --to 's3://example-nix-cache' $OUT_PATHS exec nix copy --to "s3://example-nix-cache" $OUT_PATHS
``` ```
> **Note** > **Note**
> >
> The `$OUT_PATHS` variable is a space-separated list of Nix store > The `$OUT_PATHS` variable is a space-separated list of Nix store
> paths. In this case, we expect and want the shell to perform word > paths. In this case, we expect and want the shell to perform word
> splitting to make each output path its own argument to `nix > splitting to make each output path its own argument to `nix

View file

@ -53,16 +53,18 @@ All options not listed here are passed to `nix-store
--realise`, except for `--arg` and `--attr` / `-A` which are passed to --realise`, except for `--arg` and `--attr` / `-A` which are passed to
`nix-instantiate`. `nix-instantiate`.
- [`--no-out-link`]{#opt-no-out-link}\ - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
Do not create a symlink to the output path. Note that as a result Do not create a symlink to the output path. Note that as a result
the output does not become a root of the garbage collector, and so the output does not become a root of the garbage collector, and so
might be deleted by `nix-store might be deleted by `nix-store --gc`.
--gc`.
- <span id="opt-dry-run">[`--dry-run`](#opt-dry-run)</span>
- [`--dry-run`]{#opt-dry-run}\
Show what store paths would be built or downloaded. Show what store paths would be built or downloaded.
- [`--out-link`]{#opt-out-link} / `-o` *outlink*\ - <span id="opt-out-link">[`--out-link`](#opt-out-link)</span> / `-o` *outlink*
Change the name of the symlink to the output path created from Change the name of the symlink to the output path created from
`result` to *outlink*. `result` to *outlink*.

View file

@ -22,7 +22,8 @@ This section lists the options that are common to all operations. These
options are allowed for every subcommand, though they may not always options are allowed for every subcommand, though they may not always
have an effect. have an effect.
- [`--add-root`]{#opt-add-root} *path*\ - <span id="opt-add-root">[`--add-root`](#opt-add-root)</span> *path*
Causes the result of a realisation (`--realise` and Causes the result of a realisation (`--realise` and
`--force-realise`) to be registered as a root of the garbage `--force-realise`) to be registered as a root of the garbage
collector. *path* will be created as a symlink to the resulting collector. *path* will be created as a symlink to the resulting
@ -104,10 +105,6 @@ The following flags are available:
previous build, the new output path is left in previous build, the new output path is left in
`/nix/store/name.check.` `/nix/store/name.check.`
See also the `build-repeat` configuration option, which repeats a
derivation a number of times and prevents its outputs from being
registered as “valid” in the Nix store unless they are identical.
Special exit codes: Special exit codes:
- `100`\ - `100`\

View file

@ -88,6 +88,29 @@ extension. The installer will also create `/etc/profile.d/nix.sh`.
### Linux ### Linux
If you are on Linux with systemd:
1. Remove the Nix daemon service:
```console
sudo systemctl stop nix-daemon.service
sudo systemctl disable nix-daemon.socket nix-daemon.service
sudo systemctl daemon-reload
```
1. Remove systemd service files:
```console
sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket
```
1. The installer script uses systemd-tmpfiles to create the socket directory.
You may also want to remove the configuration for that:
```console
sudo rm /etc/tmpfiles.d/nix-daemon.conf
```
Remove files created by Nix: Remove files created by Nix:
```console ```console
@ -103,16 +126,6 @@ done
sudo groupdel 30000 sudo groupdel 30000
``` ```
If you are on Linux with systemd, remove the Nix daemon service:
```console
sudo systemctl stop nix-daemon.socket
sudo systemctl stop nix-daemon.service
sudo systemctl disable nix-daemon.socket
sudo systemctl disable nix-daemon.service
sudo systemctl daemon-reload
```
There may also be references to Nix in There may also be references to Nix in
- `/etc/profile` - `/etc/profile`

View file

@ -0,0 +1,43 @@
# Release 2.12 (2022-12-06)
* On Linux, Nix can now run builds in a user namespace where they run
as root (UID 0) and have 65,536 UIDs available.
<!-- FIXME: move this to its own section about system features -->
This is primarily useful for running containers such as `systemd-nspawn`
inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn].
[nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
A build can enable this by setting the derivation attribute:
```
requiredSystemFeatures = [ "uid-range" ];
```
The `uid-range` [system feature] requires the [`auto-allocate-uids`]
setting to be enabled.
[system feature]: (../command-ref/conf-file.md#conf-system-features)
* Nix can now automatically pick UIDs for builds, removing the need to
create `nixbld*` user accounts. See [`auto-allocate-uids`].
[`auto-allocate-uids`]: (../command-ref/conf-file.md#conf-auto-allocate-uids)
* On Linux, Nix has experimental support for running builds inside a
cgroup. See
[`use-cgroups`](../command-ref/conf-file.md#conf-use-cgroups).
* `<nix/fetchurl.nix>` now accepts an additional argument `impure` which
defaults to `false`. If it is set to `true`, the `hash` and `sha256`
arguments will be ignored and the resulting derivation will have
`__impure` set to `true`, making it an impure derivation.
* If `builtins.readFile` is called on a file with context, then only
the parts of the context that appear in the content of the file are
retained. This avoids a lot of spurious errors where strings end up
having a context just because they are read from a store path
([#7260](https://github.com/NixOS/nix/pull/7260)).
* `nix build --json` now prints some statistics about top-level
derivations, such as CPU statistics when cgroups are enabled.

View file

@ -1,14 +1,6 @@
# Release X.Y (202?-??-??) # Release X.Y (202?-??-??)
* `<nix/fetchurl.nix>` now accepts an additional argument `impure` which * The `repeat` and `enforce-determinism` options have been removed
defaults to `false`. If it is set to `true`, the `hash` and `sha256` since they had been broken under many circumstances for a long time.
arguments will be ignored and the resulting derivation will have
`__impure` set to `true`, making it an impure derivation.
* If `builtins.readFile` is called on a file with context, then only the parts
of that context that appear in the content of the file are retained.
This avoids a lot of spurious errors where some benign strings end-up having
a context just because they are read from a store path
([#7260](https://github.com/NixOS/nix/pull/7260)).
* Allow explicitly selecting outputs with *store derivations* installable syntax too. * Allow explicitly selecting outputs with *store derivations* installable syntax too.

View file

@ -36,6 +36,17 @@ let
shell = "${pkgs.bashInteractive}/bin/bash"; shell = "${pkgs.bashInteractive}/bin/bash";
home = "/root"; home = "/root";
gid = 0; gid = 0;
groups = [ "root" ];
description = "System administrator";
};
nobody = {
uid = 65534;
shell = "${pkgs.shadow}/bin/nologin";
home = "/var/empty";
gid = 65534;
groups = [ "nobody" ];
description = "Unprivileged account (don't use!)";
}; };
} // lib.listToAttrs ( } // lib.listToAttrs (
@ -57,6 +68,7 @@ let
groups = { groups = {
root.gid = 0; root.gid = 0;
nixbld.gid = 30000; nixbld.gid = 30000;
nobody.gid = 65534;
}; };
userToPasswd = ( userToPasswd = (

View file

@ -9,14 +9,14 @@
let let
version = builtins.readFile ./.version + versionSuffix; officialRelease = false;
version = nixpkgs.lib.fileContents ./.version + versionSuffix;
versionSuffix = versionSuffix =
if officialRelease if officialRelease
then "" then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
officialRelease = false;
linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ];
@ -506,6 +506,12 @@
overlay = self.overlays.default; overlay = self.overlays.default;
}); });
tests.containers = (import ./tests/containers.nix rec {
system = "x86_64-linux";
inherit nixpkgs;
overlay = self.overlays.default;
});
tests.setuid = nixpkgs.lib.genAttrs tests.setuid = nixpkgs.lib.genAttrs
["i686-linux" "x86_64-linux"] ["i686-linux" "x86_64-linux"]
(system: (system:

79
maintainers/README.md Normal file
View file

@ -0,0 +1,79 @@
# Nix maintainers team
## Motivation
The goal of the team is to help other people to contribute to Nix.
## Members
- Eelco Dolstra (@edolstra) Team lead
- Théophane Hufschmitt (@thufschmitt)
- Valentin Gagarin (@fricklerhandwerk)
- Thomas Bereknyei (@tomberek)
- Robert Hensing (@roberth)
## Meeting protocol
The team meets twice a week:
- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
1. Triage issues and pull requests from the _No Status_ column (30 min)
2. Discuss issues and pull requests from the _To discuss_ column (30 min)
- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
1. Code review on pull requests from _In review_.
2. Other chores and tasks.
Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50).
## Project board protocol
The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work.
Issues on the board progress through the following states:
- No Status
Team members can add pull requests or issues to discuss or review together.
During the discussion meeting, the team triages new items.
If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_.
- To discuss
Pull requests and issues that are important and controversial are discussed by the team during discussion meetings.
This may be where the merit of the change itself or the implementation strategy is contested by a team member.
- In review
Pull requests in this column are reviewed together during work meetings.
This is both for spreading implementation knowledge and for establishing common values in code reviews.
When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member.
- Assigned for merging
One team member is assigned to each of these pull requests.
They will communicate with the authors, and make the final approval once all remaining issues are addressed.
If more substantive issues arise, the assignee can move the pull request back to _To discuss_ to involve the team again.
The process is illustrated in the following diagram:
```mermaid
flowchart TD
discuss[To discuss]
review[To review]
New --> |Disagreement on idea| discuss
New & discuss --> |Consensus on idea| review
review --> |Consensus on implementation| Assigned
Assigned --> |Implementation issues arise| review
Assigned --> |Remaining issues fixed| Merged
```

View file

@ -28,7 +28,7 @@
<key>SoftResourceLimits</key> <key>SoftResourceLimits</key>
<dict> <dict>
<key>NumberOfFiles</key> <key>NumberOfFiles</key>
<integer>4096</integer> <integer>1048576</integer>
</dict> </dict>
</dict> </dict>
</plist> </plist>

View file

@ -9,7 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
[Service] [Service]
ExecStart=@@bindir@/nix-daemon nix-daemon --daemon ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
KillMode=process KillMode=process
LimitNOFILE=4096 LimitNOFILE=1048576
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

View file

@ -97,13 +97,10 @@ is_os_darwin() {
} }
contact_us() { contact_us() {
echo "You can open an issue at https://github.com/nixos/nix/issues" echo "You can open an issue at"
echo "https://github.com/NixOS/nix/issues/new?labels=installer&template=installer.md"
echo "" echo ""
echo "Or feel free to contact the team:" echo "Or get in touch with the community: https://nixos.org/community"
echo " - Matrix: #nix:nixos.org"
echo " - IRC: in #nixos on irc.libera.chat"
echo " - twitter: @nixos_org"
echo " - forum: https://discourse.nixos.org"
} }
get_help() { get_help() {
echo "We'd love to help if you need it." echo "We'd love to help if you need it."

View file

@ -958,7 +958,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
case Realise::Outputs: { case Realise::Outputs: {
if (settings.printMissing) if (settings.printMissing)
printMissing(store, pathsToBuild, lvlInfo); printMissing(store, pathsToBuild, lvlInfo);
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success()) if (!buildResult.success())

View file

@ -270,6 +270,7 @@ void NixRepl::mainLoop()
// ctrl-D should exit the debugger. // ctrl-D should exit the debugger.
state->debugStop = false; state->debugStop = false;
state->debugQuit = true; state->debugQuit = true;
logger->cout("");
break; break;
} }
try { try {

View file

@ -7,7 +7,6 @@
#include "globals.hh" #include "globals.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "json.hh"
#include "function-trace.hh" #include "function-trace.hh"
#include <algorithm> #include <algorithm>
@ -21,6 +20,7 @@
#include <functional> #include <functional>
#include <sys/resource.h> #include <sys/resource.h>
#include <nlohmann/json.hpp>
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
@ -35,6 +35,8 @@
#endif #endif
using json = nlohmann::json;
namespace nix { namespace nix {
static char * allocString(size_t size) static char * allocString(size_t size)
@ -69,15 +71,11 @@ static char * dupString(const char * s)
// empty string. // empty string.
static const char * makeImmutableStringWithLen(const char * s, size_t size) static const char * makeImmutableStringWithLen(const char * s, size_t size)
{ {
char * t;
if (size == 0) if (size == 0)
return ""; return "";
#if HAVE_BOEHMGC auto t = allocString(size + 1);
t = GC_STRNDUP(s, size); memcpy(t, s, size);
#else t[size] = 0;
t = strndup(s, size);
#endif
if (!t) throw std::bad_alloc();
return t; return t;
} }
@ -1648,7 +1646,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
auto dts = debugRepl auto dts = debugRepl
? makeDebugTraceStacker( ? makeDebugTraceStacker(
*this, *lambda.body, env2, positions[lambda.pos], *this, *lambda.body, env2, positions[lambda.pos],
"while evaluating %s", "while calling %s",
lambda.name lambda.name
? concatStrings("'", symbols[lambda.name], "'") ? concatStrings("'", symbols[lambda.name], "'")
: "anonymous lambda") : "anonymous lambda")
@ -1657,7 +1655,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
lambda.body->eval(*this, env2, vCur); lambda.body->eval(*this, env2, vCur);
} catch (Error & e) { } catch (Error & e) {
if (loggerSettings.showTrace.get()) { if (loggerSettings.showTrace.get()) {
addErrorTrace(e, lambda.pos, "while evaluating %s", addErrorTrace(e, lambda.pos, "while calling %s",
(lambda.name (lambda.name
? concatStrings("'", symbols[lambda.name], "'") ? concatStrings("'", symbols[lambda.name], "'")
: "anonymous lambda")); : "anonymous lambda"));
@ -2441,97 +2439,97 @@ void EvalState::printStats()
std::fstream fs; std::fstream fs;
if (outPath != "-") if (outPath != "-")
fs.open(outPath, std::fstream::out); fs.open(outPath, std::fstream::out);
JSONObject topObj(outPath == "-" ? std::cerr : fs, true); json topObj = json::object();
topObj.attr("cpuTime",cpuTime); topObj["cpuTime"] = cpuTime;
{ topObj["envs"] = {
auto envs = topObj.object("envs"); {"number", nrEnvs},
envs.attr("number", nrEnvs); {"elements", nrValuesInEnvs},
envs.attr("elements", nrValuesInEnvs); {"bytes", bEnvs},
envs.attr("bytes", bEnvs); };
} topObj["list"] = {
{ {"elements", nrListElems},
auto lists = topObj.object("list"); {"bytes", bLists},
lists.attr("elements", nrListElems); {"concats", nrListConcats},
lists.attr("bytes", bLists); };
lists.attr("concats", nrListConcats); topObj["values"] = {
} {"number", nrValues},
{ {"bytes", bValues},
auto values = topObj.object("values"); };
values.attr("number", nrValues); topObj["symbols"] = {
values.attr("bytes", bValues); {"number", symbols.size()},
} {"bytes", symbols.totalSize()},
{ };
auto syms = topObj.object("symbols"); topObj["sets"] = {
syms.attr("number", symbols.size()); {"number", nrAttrsets},
syms.attr("bytes", symbols.totalSize()); {"bytes", bAttrsets},
} {"elements", nrAttrsInAttrsets},
{ };
auto sets = topObj.object("sets"); topObj["sizes"] = {
sets.attr("number", nrAttrsets); {"Env", sizeof(Env)},
sets.attr("bytes", bAttrsets); {"Value", sizeof(Value)},
sets.attr("elements", nrAttrsInAttrsets); {"Bindings", sizeof(Bindings)},
} {"Attr", sizeof(Attr)},
{ };
auto sizes = topObj.object("sizes"); topObj["nrOpUpdates"] = nrOpUpdates;
sizes.attr("Env", sizeof(Env)); topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
sizes.attr("Value", sizeof(Value)); topObj["nrThunks"] = nrThunks;
sizes.attr("Bindings", sizeof(Bindings)); topObj["nrAvoided"] = nrAvoided;
sizes.attr("Attr", sizeof(Attr)); topObj["nrLookups"] = nrLookups;
} topObj["nrPrimOpCalls"] = nrPrimOpCalls;
topObj.attr("nrOpUpdates", nrOpUpdates); topObj["nrFunctionCalls"] = nrFunctionCalls;
topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
topObj.attr("nrThunks", nrThunks);
topObj.attr("nrAvoided", nrAvoided);
topObj.attr("nrLookups", nrLookups);
topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
topObj.attr("nrFunctionCalls", nrFunctionCalls);
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
{ topObj["gc"] = {
auto gc = topObj.object("gc"); {"heapSize", heapSize},
gc.attr("heapSize", heapSize); {"totalBytes", totalBytes},
gc.attr("totalBytes", totalBytes); };
}
#endif #endif
if (countCalls) { if (countCalls) {
topObj["primops"] = primOpCalls;
{ {
auto obj = topObj.object("primops"); auto& list = topObj["functions"];
for (auto & i : primOpCalls) list = json::array();
obj.attr(i.first, i.second);
}
{
auto list = topObj.list("functions");
for (auto & [fun, count] : functionCalls) { for (auto & [fun, count] : functionCalls) {
auto obj = list.object(); json obj = json::object();
if (fun->name) if (fun->name)
obj.attr("name", (std::string_view) symbols[fun->name]); obj["name"] = (std::string_view) symbols[fun->name];
else else
obj.attr("name", nullptr); obj["name"] = nullptr;
if (auto pos = positions[fun->pos]) { if (auto pos = positions[fun->pos]) {
obj.attr("file", (std::string_view) pos.file); obj["file"] = (std::string_view) pos.file;
obj.attr("line", pos.line); obj["line"] = pos.line;
obj.attr("column", pos.column); obj["column"] = pos.column;
} }
obj.attr("count", count); obj["count"] = count;
list.push_back(obj);
} }
} }
{ {
auto list = topObj.list("attributes"); auto list = topObj["attributes"];
list = json::array();
for (auto & i : attrSelects) { for (auto & i : attrSelects) {
auto obj = list.object(); json obj = json::object();
if (auto pos = positions[i.first]) { if (auto pos = positions[i.first]) {
obj.attr("file", (const std::string &) pos.file); obj["file"] = (const std::string &) pos.file;
obj.attr("line", pos.line); obj["line"] = pos.line;
obj.attr("column", pos.column); obj["column"] = pos.column;
} }
obj.attr("count", i.second); obj["count"] = i.second;
list.push_back(obj);
} }
} }
} }
if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
auto list = topObj.list("symbols"); // XXX: overrides earlier assignment
symbols.dump([&](const std::string & s) { list.elem(s); }); topObj["symbols"] = json::array();
auto &list = topObj["symbols"];
symbols.dump([&](const std::string & s) { list.emplace_back(s); });
}
if (outPath == "-") {
std::cerr << topObj.dump(2) << std::endl;
} else {
fs << topObj.dump(2) << std::endl;
} }
} }
} }

View file

@ -8,12 +8,12 @@
#include "references.hh" #include "references.hh"
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "json.hh"
#include "value-to-json.hh" #include "value-to-json.hh"
#include "value-to-xml.hh" #include "value-to-xml.hh"
#include "primops.hh" #include "primops.hh"
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@ -1011,6 +1011,7 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val
derivation. */ derivation. */
static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{ {
using nlohmann::json;
state.forceAttrs(*args[0], pos); state.forceAttrs(*args[0], pos);
/* Figure out the name first (for stack backtraces). */ /* Figure out the name first (for stack backtraces). */
@ -1032,11 +1033,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
} }
/* Check whether attributes should be passed as a JSON file. */ /* Check whether attributes should be passed as a JSON file. */
std::ostringstream jsonBuf; std::optional<json> jsonObject;
std::unique_ptr<JSONObject> jsonObject;
attr = args[0]->attrs->find(state.sStructuredAttrs); attr = args[0]->attrs->find(state.sStructuredAttrs);
if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
jsonObject = std::make_unique<JSONObject>(jsonBuf); jsonObject = json::object();
/* Check whether null attributes should be ignored. */ /* Check whether null attributes should be ignored. */
bool ignoreNulls = false; bool ignoreNulls = false;
@ -1138,8 +1138,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
if (i->name == state.sStructuredAttrs) continue; if (i->name == state.sStructuredAttrs) continue;
auto placeholder(jsonObject->placeholder(key)); (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
printValueAsJSON(state, true, *i->value, pos, placeholder, context);
if (i->name == state.sBuilder) if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, posDrvName); drv.builder = state.forceString(*i->value, context, posDrvName);
@ -1183,8 +1182,8 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
} }
if (jsonObject) { if (jsonObject) {
drv.env.emplace("__json", jsonObject->dump());
jsonObject.reset(); jsonObject.reset();
drv.env.emplace("__json", jsonBuf.str());
} }
/* Everything in the context of the strings in the derivation /* Everything in the context of the strings in the derivation
@ -2421,12 +2420,18 @@ static RegisterPrimOp primop_listToAttrs({
Construct a set from a list specifying the names and values of each Construct a set from a list specifying the names and values of each
attribute. Each element of the list should be a set consisting of a attribute. Each element of the list should be a set consisting of a
string-valued attribute `name` specifying the name of the attribute, string-valued attribute `name` specifying the name of the attribute,
and an attribute `value` specifying its value. Example: and an attribute `value` specifying its value.
In case of duplicate occurrences of the same name, the first
takes precedence.
Example:
```nix ```nix
builtins.listToAttrs builtins.listToAttrs
[ { name = "foo"; value = 123; } [ { name = "foo"; value = 123; }
{ name = "bar"; value = 456; } { name = "bar"; value = 456; }
{ name = "bar"; value = 420; }
] ]
``` ```

View file

@ -1,84 +1,82 @@
#include "value-to-json.hh" #include "value-to-json.hh"
#include "json.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
using json = nlohmann::json;
void printValueAsJSON(EvalState & state, bool strict, json printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore) Value & v, const PosIdx pos, PathSet & context, bool copyToStore)
{ {
checkInterrupt(); checkInterrupt();
if (strict) state.forceValue(v, pos); if (strict) state.forceValue(v, pos);
json out;
switch (v.type()) { switch (v.type()) {
case nInt: case nInt:
out.write(v.integer); out = v.integer;
break; break;
case nBool: case nBool:
out.write(v.boolean); out = v.boolean;
break; break;
case nString: case nString:
copyContext(v, context); copyContext(v, context);
out.write(v.string.s); out = v.string.s;
break; break;
case nPath: case nPath:
if (copyToStore) if (copyToStore)
out.write(state.copyPathToStore(context, v.path)); out = state.copyPathToStore(context, v.path);
else else
out.write(v.path); out = v.path;
break; break;
case nNull: case nNull:
out.write(nullptr);
break; break;
case nAttrs: { case nAttrs: {
auto maybeString = state.tryAttrsToString(pos, v, context, false, false); auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
if (maybeString) { if (maybeString) {
out.write(*maybeString); out = *maybeString;
break; break;
} }
auto i = v.attrs->find(state.sOutPath); auto i = v.attrs->find(state.sOutPath);
if (i == v.attrs->end()) { if (i == v.attrs->end()) {
auto obj(out.object()); out = json::object();
StringSet names; StringSet names;
for (auto & j : *v.attrs) for (auto & j : *v.attrs)
names.emplace(state.symbols[j.name]); names.emplace(state.symbols[j.name]);
for (auto & j : names) { for (auto & j : names) {
Attr & a(*v.attrs->find(state.symbols.create(j))); Attr & a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j)); out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore);
printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore);
} }
} else } else
printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore); return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore);
break; break;
} }
case nList: { case nList: {
auto list(out.list()); out = json::array();
for (auto elem : v.listItems()) { for (auto elem : v.listItems())
auto placeholder(list.placeholder()); out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore));
printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore);
}
break; break;
} }
case nExternal: case nExternal:
v.external->printValueAsJSON(state, strict, out, context, copyToStore); return v.external->printValueAsJSON(state, strict, context, copyToStore);
break; break;
case nFloat: case nFloat:
out.write(v.fpoint); out = v.fpoint;
break; break;
case nThunk: case nThunk:
@ -91,17 +89,17 @@ void printValueAsJSON(EvalState & state, bool strict,
state.debugThrowLastTrace(e); state.debugThrowLastTrace(e);
throw e; throw e;
} }
return out;
} }
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore)
{ {
JSONPlaceholder out(str); str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
printValueAsJSON(state, strict, v, pos, out, context, copyToStore);
} }
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context, bool copyToStore) const PathSet & context, bool copyToStore) const
{ {
state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
} }

View file

@ -5,13 +5,12 @@
#include <string> #include <string>
#include <map> #include <map>
#include <nlohmann/json_fwd.hpp>
namespace nix { namespace nix {
class JSONPlaceholder; nlohmann::json printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true);
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true);
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true);

View file

@ -7,6 +7,7 @@
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
#include <gc/gc_allocator.h> #include <gc/gc_allocator.h>
#endif #endif
#include <nlohmann/json_fwd.hpp>
namespace nix { namespace nix {
@ -62,7 +63,6 @@ class StorePath;
class Store; class Store;
class EvalState; class EvalState;
class XMLWriter; class XMLWriter;
class JSONPlaceholder;
typedef int64_t NixInt; typedef int64_t NixInt;
@ -98,8 +98,8 @@ class ExternalValueBase
virtual bool operator ==(const ExternalValueBase & b) const; virtual bool operator ==(const ExternalValueBase & b) const;
/* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
virtual void printValueAsJSON(EvalState & state, bool strict, virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const; PathSet & context, bool copyToStore = true) const;
/* Print the value as XML. Defaults to unevaluated */ /* Print the value as XML. Defaults to unevaluated */
virtual void printValueAsXML(EvalState & state, bool strict, bool location, virtual void printValueAsXML(EvalState & state, bool strict, bool location,

View file

@ -180,10 +180,12 @@ public:
auto machineName = getS(fields, 1); auto machineName = getS(fields, 1);
if (machineName != "") if (machineName != "")
i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName);
auto curRound = getI(fields, 2);
auto nrRounds = getI(fields, 3); // Used to be curRound and nrRounds, but the
if (nrRounds != 1) // implementation was broken for a long time.
i->s += fmt(" (round %d/%d)", curRound, nrRounds); if (getI(fields, 2) != 1 || getI(fields, 3) != 1) {
throw Error("log message indicated repeating builds, but this is not currently implemented");
}
i->name = DrvName(name).name; i->name = DrvName(name).name;
} }

View file

@ -9,7 +9,6 @@
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "json.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "callback.hh" #include "callback.hh"
@ -194,19 +193,12 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
/* Optionally write a JSON file containing a listing of the /* Optionally write a JSON file containing a listing of the
contents of the NAR. */ contents of the NAR. */
if (writeNARListing) { if (writeNARListing) {
std::ostringstream jsonOut; nlohmann::json j = {
{"version", 1},
{"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
};
{ upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1);
{
auto res = jsonRoot.placeholder("root");
listNar(res, ref<FSAccessor>(narAccessor), "", true);
}
}
upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
} }
/* Optionally maintain an index of DWARF debug info files /* Optionally maintain an index of DWARF debug info files

View file

@ -5,7 +5,7 @@
#include <string> #include <string>
#include <chrono> #include <chrono>
#include <optional>
namespace nix { namespace nix {
@ -78,6 +78,9 @@ struct BuildResult
was repeated). */ was repeated). */
time_t startTime = 0, stopTime = 0; time_t startTime = 0, stopTime = 0;
/* User and system CPU time the build took. */
std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
bool success() bool success()
{ {
return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;

View file

@ -7,7 +7,6 @@
#include "finally.hh" #include "finally.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh" #include "archive.hh"
#include "json.hh"
#include "compression.hh" #include "compression.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
@ -502,6 +501,14 @@ void DerivationGoal::inputsRealised()
now-known results of dependencies. If so, we become a now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */ stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
if (!attempt) {
/* TODO (impure derivations-induced tech debt) (see below):
The above attempt should have found it, but because we manage
inputDrvOutputs statefully, sometimes it gets out of sync with
the real source of truth (store). So we query the store
directly if there's a problem. */
attempt = fullDrv.tryResolve(worker.store);
}
assert(attempt); assert(attempt);
Derivation drvResolved { *std::move(attempt) }; Derivation drvResolved { *std::move(attempt) };
@ -564,10 +571,6 @@ void DerivationGoal::inputsRealised()
/* What type of derivation are we building? */ /* What type of derivation are we building? */
derivationType = drv->type(); derivationType = drv->type();
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build /* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a slot to become available, since we don't need one if there is a
build hook. */ build hook. */
@ -582,12 +585,11 @@ void DerivationGoal::started()
auto msg = fmt( auto msg = fmt(
buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmRepair ? "repairing outputs of '%s'" :
buildMode == bmCheck ? "checking outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" :
nrRounds > 1 ? "building '%s' (round %d/%d)" : "building '%s'", worker.store.printStorePath(drvPath));
"building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
fmt("building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath));
if (hook) msg += fmt(" on '%s'", machineName); if (hook) msg += fmt(" on '%s'", machineName);
act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1});
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
worker.updateProgress(); worker.updateProgress();
} }
@ -887,6 +889,14 @@ void DerivationGoal::buildDone()
cleanupPostChildKill(); cleanupPostChildKill();
if (buildResult.cpuUser && buildResult.cpuSystem) {
debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs",
worker.store.printStorePath(drvPath),
status,
((double) buildResult.cpuUser->count()) / 1000000,
((double) buildResult.cpuSystem->count()) / 1000000);
}
bool diskFull = false; bool diskFull = false;
try { try {
@ -933,14 +943,6 @@ void DerivationGoal::buildDone()
cleanupPostOutputsRegisteredModeNonCheck(); cleanupPostOutputsRegisteredModeNonCheck();
/* Repeat the build if necessary. */
if (curRound++ < nrRounds) {
outputLocks.unlock();
state = &DerivationGoal::tryToBuild;
worker.wakeUp(shared_from_this());
return;
}
/* It is now safe to delete the lock files, since all future /* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will lockers will see that the output paths are valid; they will
not create new lock files with the same names as the old not create new lock files with the same names as the old
@ -1001,22 +1003,34 @@ void DerivationGoal::resolvedFinished()
throw Error( throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
worker.store.printStorePath(drvPath), wantedOutput); worker.store.printStorePath(drvPath), wantedOutput);
auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
if (!realisation) auto realisation = [&]{
throw Error( auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", if (take1) return *take1;
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
/* The above `get` should work. But sateful tracking of
outputs in resolvedResult, this can get out of sync with the
store, which is our actual source of truth. For now we just
check the store directly if it fails. */
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
if (take2) return *take2;
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
}();
if (drv->type().isPure()) { if (drv->type().isPure()) {
auto newRealisation = *realisation; auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
newRealisation.signatures.clear(); newRealisation.signatures.clear();
if (!drv->type().isFixed()) if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
signRealisation(newRealisation); signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
} }
outputPaths.insert(realisation->outPath); outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation->id, *realisation); builtOutputs.emplace(realisation.id, realisation);
} }
runPostBuildHook( runPostBuildHook(

View file

@ -115,11 +115,6 @@ struct DerivationGoal : public Goal
BuildMode buildMode; BuildMode buildMode;
/* The current round, if we're building multiple times. */
size_t curRound = 1;
size_t nrRounds;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act; std::unique_ptr<Activity> act;

View file

@ -8,13 +8,13 @@
#include "finally.hh" #include "finally.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh" #include "archive.hh"
#include "json.hh"
#include "compression.hh" #include "compression.hh"
#include "daemon.hh" #include "daemon.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "callback.hh" #include "callback.hh"
#include "json-utils.hh" #include "json-utils.hh"
#include "cgroup.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>
@ -56,6 +56,7 @@
#include <pwd.h> #include <pwd.h>
#include <grp.h> #include <grp.h>
#include <iostream>
namespace nix { namespace nix {
@ -129,26 +130,44 @@ void LocalDerivationGoal::killChild()
if (pid != -1) { if (pid != -1) {
worker.childTerminated(this); worker.childTerminated(this);
if (buildUser) { /* If we're using a build user, then there is a tricky race
/* If we're using a build user, then there is a tricky condition: if we kill the build user before the child has
race condition: if we kill the build user before the done its setuid() to the build user uid, then it won't be
child has done its setuid() to the build user uid, then killed, and we'll potentially lock up in pid.wait(). So
it won't be killed, and we'll potentially lock up in also send a conventional kill to the child. */
pid.wait(). So also send a conventional kill to the ::kill(-pid, SIGKILL); /* ignore the result */
child. */
::kill(-pid, SIGKILL); /* ignore the result */
buildUser->kill();
pid.wait();
} else
pid.kill();
assert(pid == -1); killSandbox(true);
pid.wait();
} }
DerivationGoal::killChild(); DerivationGoal::killChild();
} }
void LocalDerivationGoal::killSandbox(bool getStats)
{
if (cgroup) {
#if __linux__
auto stats = destroyCgroup(*cgroup);
if (getStats) {
buildResult.cpuUser = stats.cpuUser;
buildResult.cpuSystem = stats.cpuSystem;
}
#else
abort();
#endif
}
else if (buildUser) {
auto uid = buildUser->getUID();
assert(uid != 0);
killUser(uid);
}
}
void LocalDerivationGoal::tryLocalBuild() { void LocalDerivationGoal::tryLocalBuild() {
unsigned int curBuilds = worker.getNrLocalBuilds(); unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) { if (curBuilds >= settings.maxBuildJobs) {
@ -158,28 +177,46 @@ void LocalDerivationGoal::tryLocalBuild() {
return; return;
} }
/* If `build-users-group' is not empty, then we have to build as /* Are we doing a chroot build? */
one of the members of that group. */ {
if (settings.buildUsersGroup != "" && getuid() == 0) { auto noChroot = parsedDrv->getBoolAttr("__noChroot");
#if defined(__linux__) || defined(__APPLE__) if (settings.sandboxMode == smEnabled) {
if (!buildUser) buildUser = std::make_unique<UserLock>(); if (noChroot)
throw Error("derivation '%s' has '__noChroot' set, "
"but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
#if __APPLE__
if (additionalSandboxProfile != "")
throw Error("derivation '%s' specifies a sandbox profile, "
"but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
#endif
useChroot = true;
}
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = derivationType.isSandboxed() && !noChroot;
}
if (buildUser->findFreeUser()) { auto & localStore = getLocalStore();
/* Make sure that no other processes are executing under this if (localStore.storeDir != localStore.realStoreDir.get()) {
uid. */ #if __linux__
buildUser->kill(); useChroot = true;
} else { #else
throw Error("building using a diverted store is not supported on this platform");
#endif
}
if (useBuildUsers()) {
if (!buildUser)
buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
if (!buildUser) {
if (!actLock) if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
return; return;
} }
#else
/* Don't know how to block the creation of setuid/setgid
binaries on this platform. */
throw Error("build users are not supported on this platform for security reasons");
#endif
} }
actLock.reset(); actLock.reset();
@ -270,7 +307,7 @@ void LocalDerivationGoal::cleanupPostChildKill()
malicious user from leaving behind a process that keeps files malicious user from leaving behind a process that keeps files
open and modifies them after they have been chown'ed to open and modifies them after they have been chown'ed to
root. */ root. */
if (buildUser) buildUser->kill(); killSandbox(true);
/* Terminate the recursive Nix daemon. */ /* Terminate the recursive Nix daemon. */
stopDaemon(); stopDaemon();
@ -363,6 +400,64 @@ static void linkOrCopy(const Path & from, const Path & to)
void LocalDerivationGoal::startBuilder() void LocalDerivationGoal::startBuilder()
{ {
if ((buildUser && buildUser->getUIDCount() != 1)
#if __linux__
|| settings.useCgroups
#endif
)
{
#if __linux__
settings.requireExperimentalFeature(Xp::Cgroups);
auto cgroupFS = getCgroupFS();
if (!cgroupFS)
throw Error("cannot determine the cgroups file system");
auto ourCgroups = getCgroups("/proc/self/cgroup");
auto ourCgroup = ourCgroups[""];
if (ourCgroup == "")
throw Error("cannot determine cgroup name from /proc/self/cgroup");
auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup);
if (!pathExists(ourCgroupPath))
throw Error("expected cgroup directory '%s'", ourCgroupPath);
static std::atomic<unsigned int> counter{0};
cgroup = buildUser
? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID())
: fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++);
debug("using cgroup '%s'", *cgroup);
/* When using a build user, record the cgroup we used for that
user so that if we got interrupted previously, we can kill
any left-over cgroup first. */
if (buildUser) {
auto cgroupsDir = settings.nixStateDir + "/cgroups";
createDirs(cgroupsDir);
auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID());
if (pathExists(cgroupFile)) {
auto prevCgroup = readFile(cgroupFile);
destroyCgroup(prevCgroup);
}
writeFile(cgroupFile, *cgroup);
}
#else
throw Error("cgroups are not supported on this platform");
#endif
}
/* Make sure that no other processes are executing under the
sandbox uids. This must be done before any chownToBuilder()
calls. */
killSandbox(false);
/* Right platform? */ /* Right platform? */
if (!parsedDrv->canBuildLocally(worker.store)) if (!parsedDrv->canBuildLocally(worker.store))
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
@ -376,35 +471,6 @@ void LocalDerivationGoal::startBuilder()
additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
#endif #endif
/* Are we doing a chroot build? */
{
auto noChroot = parsedDrv->getBoolAttr("__noChroot");
if (settings.sandboxMode == smEnabled) {
if (noChroot)
throw Error("derivation '%s' has '__noChroot' set, "
"but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
#if __APPLE__
if (additionalSandboxProfile != "")
throw Error("derivation '%s' specifies a sandbox profile, "
"but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
#endif
useChroot = true;
}
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = derivationType.isSandboxed() && !noChroot;
}
auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir.get()) {
#if __linux__
useChroot = true;
#else
throw Error("building using a diverted store is not supported on this platform");
#endif
}
/* Create a temporary directory where the build will take /* Create a temporary directory where the build will take
place. */ place. */
tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
@ -580,10 +646,11 @@ void LocalDerivationGoal::startBuilder()
printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
if (mkdir(chrootRootDir.c_str(), 0750) == -1) // FIXME: make this 0700
if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
throw SysError("cannot create '%1%'", chrootRootDir); throw SysError("cannot create '%1%'", chrootRootDir);
if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1)
throw SysError("cannot change ownership of '%1%'", chrootRootDir); throw SysError("cannot change ownership of '%1%'", chrootRootDir);
/* Create a writable /tmp in the chroot. Many builders need /* Create a writable /tmp in the chroot. Many builders need
@ -597,6 +664,10 @@ void LocalDerivationGoal::startBuilder()
nobody account. The latter is kind of a hack to support nobody account. The latter is kind of a hack to support
Samba-in-QEMU. */ Samba-in-QEMU. */
createDirs(chrootRootDir + "/etc"); createDirs(chrootRootDir + "/etc");
chownToBuilder(chrootRootDir + "/etc");
if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
/* Declare the build user's group so that programs get a consistent /* Declare the build user's group so that programs get a consistent
view of the system (e.g., "id -gn"). */ view of the system (e.g., "id -gn"). */
@ -647,12 +718,28 @@ void LocalDerivationGoal::startBuilder()
dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
} }
#elif __APPLE__ if (cgroup) {
/* We don't really have any parent prep work to do (yet?) if (mkdir(cgroup->c_str(), 0755) != 0)
All work happens in the child, instead. */ throw SysError("creating cgroup '%s'", *cgroup);
chownToBuilder(*cgroup);
chownToBuilder(*cgroup + "/cgroup.procs");
chownToBuilder(*cgroup + "/cgroup.threads");
//chownToBuilder(*cgroup + "/cgroup.subtree_control");
}
#else #else
throw Error("sandboxing builds is not supported on this platform"); if (parsedDrv->useUidRange())
throw Error("feature 'uid-range' is not supported on this platform");
#if __APPLE__
/* We don't really have any parent prep work to do (yet?)
All work happens in the child, instead. */
#else
throw Error("sandboxing builds is not supported on this platform");
#endif
#endif #endif
} else {
if (parsedDrv->useUidRange())
throw Error("feature 'uid-range' is only supported in sandboxed builds");
} }
if (needsHashRewrite() && pathExists(homeDir)) if (needsHashRewrite() && pathExists(homeDir))
@ -913,14 +1000,16 @@ void LocalDerivationGoal::startBuilder()
the calling user (if build users are disabled). */ the calling user (if build users are disabled). */
uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1;
writeFile("/proc/" + std::to_string(pid) + "/uid_map", writeFile("/proc/" + std::to_string(pid) + "/uid_map",
fmt("%d %d 1", sandboxUid(), hostUid)); fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); if (!buildUser || buildUser->getUIDCount() == 1)
writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
writeFile("/proc/" + std::to_string(pid) + "/gid_map", writeFile("/proc/" + std::to_string(pid) + "/gid_map",
fmt("%d %d 1", sandboxGid(), hostGid)); fmt("%d %d %d", sandboxGid(), hostGid, nrIds));
} else { } else {
debug("note: not using a user namespace"); debug("note: not using a user namespace");
if (!buildUser) if (!buildUser)
@ -947,6 +1036,10 @@ void LocalDerivationGoal::startBuilder()
throw SysError("getting sandbox user namespace"); throw SysError("getting sandbox user namespace");
} }
/* Move the child into its own cgroup. */
if (cgroup)
writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
/* Signal the builder that we've updated its user namespace. */ /* Signal the builder that we've updated its user namespace. */
writeFull(userNamespaceSync.writeSide.get(), "1"); writeFull(userNamespaceSync.writeSide.get(), "1");
@ -1779,6 +1872,13 @@ void LocalDerivationGoal::runChild()
if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
throw SysError("mounting /proc"); throw SysError("mounting /proc");
/* Mount sysfs on /sys. */
if (buildUser && buildUser->getUIDCount() != 1) {
createDirs(chrootRootDir + "/sys");
if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
throw SysError("mounting /sys");
}
/* Mount a new tmpfs on /dev/shm to ensure that whatever /* Mount a new tmpfs on /dev/shm to ensure that whatever
the builder puts in /dev/shm is cleaned up automatically. */ the builder puts in /dev/shm is cleaned up automatically. */
if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
@ -1821,6 +1921,12 @@ void LocalDerivationGoal::runChild()
if (unshare(CLONE_NEWNS) == -1) if (unshare(CLONE_NEWNS) == -1)
throw SysError("unsharing mount namespace"); throw SysError("unsharing mount namespace");
/* Unshare the cgroup namespace. This means
/proc/self/cgroup will show the child's cgroup as '/'
rather than whatever it is in the parent. */
if (cgroup && unshare(CLONE_NEWCGROUP) == -1)
throw SysError("unsharing cgroup namespace");
/* Do the chroot(). */ /* Do the chroot(). */
if (chdir(chrootRootDir.c_str()) == -1) if (chdir(chrootRootDir.c_str()) == -1)
throw SysError("cannot change directory to '%1%'", chrootRootDir); throw SysError("cannot change directory to '%1%'", chrootRootDir);
@ -1906,9 +2012,8 @@ void LocalDerivationGoal::runChild()
if (setUser && buildUser) { if (setUser && buildUser) {
/* Preserve supplementary groups of the build user, to allow /* Preserve supplementary groups of the build user, to allow
admins to specify groups such as "kvm". */ admins to specify groups such as "kvm". */
if (!buildUser->getSupplementaryGIDs().empty() && auto gids = buildUser->getSupplementaryGIDs();
setgroups(buildUser->getSupplementaryGIDs().size(), if (setgroups(gids.size(), gids.data()) == -1)
buildUser->getSupplementaryGIDs().data()) == -1)
throw SysError("cannot set supplementary groups of build user"); throw SysError("cannot set supplementary groups of build user");
if (setgid(buildUser->getGID()) == -1 || if (setgid(buildUser->getGID()) == -1 ||
@ -2155,7 +2260,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
InodesSeen inodesSeen; InodesSeen inodesSeen;
Path checkSuffix = ".check"; Path checkSuffix = ".check";
bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
std::exception_ptr delayedException; std::exception_ptr delayedException;
@ -2237,7 +2341,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* Canonicalise first. This ensures that the path we're /* Canonicalise first. This ensures that the path we're
rewriting doesn't contain a hard link to /etc/shadow or rewriting doesn't contain a hard link to /etc/shadow or
something like that. */ something like that. */
canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); canonicalisePathMetaData(
actualPath,
buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt,
inodesSeen);
debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
@ -2330,6 +2437,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
sink.s = rewriteStrings(sink.s, outputRewrites); sink.s = rewriteStrings(sink.s, outputRewrites);
StringSource source(sink.s); StringSource source(sink.s);
restorePath(actualPath, source); restorePath(actualPath, source);
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, {}, inodesSeen);
} }
}; };
@ -2492,7 +2603,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* FIXME: set proper permissions in restorePath() so /* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */ we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen); canonicalisePathMetaData(actualPath, {}, inodesSeen);
/* Calculate where we'll move the output files. In the checking case we /* Calculate where we'll move the output files. In the checking case we
will leave leave them where they are, for now, rather than move to will leave leave them where they are, for now, rather than move to
@ -2576,10 +2687,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
} }
if (curRound == nrRounds) { localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() worker.markContentsGood(newInfo.path);
worker.markContentsGood(newInfo.path);
}
newInfo.deriver = drvPath; newInfo.deriver = drvPath;
newInfo.ultimate = true; newInfo.ultimate = true;
@ -2608,61 +2717,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* Apply output checks. */ /* Apply output checks. */
checkOutputs(infos); checkOutputs(infos);
/* Compare the result with the previous round, and report which
path is different, if any.*/
if (curRound > 1 && prevInfos != infos) {
assert(prevInfos.size() == infos.size());
for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
if (!(*i == *j)) {
buildResult.isNonDeterministic = true;
Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
bool prevExists = keepPreviousRound && pathExists(prev);
hintformat hint = prevExists
? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
: hintfmt("output '%s' of '%s' differs from previous round",
worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
handleDiffHook(
buildUser ? buildUser->getUID() : getuid(),
buildUser ? buildUser->getGID() : getgid(),
prev, worker.store.printStorePath(i->second.path),
worker.store.printStorePath(drvPath), tmpDir);
if (settings.enforceDeterminism)
throw NotDeterministic(hint);
printError(hint);
curRound = nrRounds; // we know enough, bail out early
}
}
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
for (auto & [_, outputStorePath] : finalOutputs) {
auto path = worker.store.printStorePath(outputStorePath);
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
renameFile(path, dst);
}
}
if (curRound < nrRounds) {
prevInfos = std::move(infos);
return {};
}
/* Remove the .check directories if we're done. FIXME: keep them
if the result was not determistic? */
if (curRound == nrRounds) {
for (auto & [_, outputStorePath] : finalOutputs) {
Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
deletePath(prev);
}
}
/* Register each output path as valid, and register the sets of /* Register each output path as valid, and register the sets of
paths referenced by each of them. If there are cycles in the paths referenced by each of them. If there are cycles in the
outputs, this will fail. */ outputs, this will fail. */

View file

@ -15,6 +15,9 @@ struct LocalDerivationGoal : public DerivationGoal
/* The process ID of the builder. */ /* The process ID of the builder. */
Pid pid; Pid pid;
/* The cgroup of the builder, if any. */
std::optional<Path> cgroup;
/* The temporary directory. */ /* The temporary directory. */
Path tmpDir; Path tmpDir;
@ -92,8 +95,8 @@ struct LocalDerivationGoal : public DerivationGoal
result. */ result. */
std::map<Path, ValidPathInfo> prevInfos; std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); }
const static Path homeDir; const static Path homeDir;
@ -197,6 +200,10 @@ struct LocalDerivationGoal : public DerivationGoal
/* Forcibly kill the child process, if any. */ /* Forcibly kill the child process, if any. */
void killChild() override; void killChild() override;
/* Kill any processes running under the build user UID or in the
cgroup of the build. */
void killSandbox(bool getStats);
/* Create alternative path calculated from but distinct from the /* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths) input, so we can avoid overwriting outputs (or other store paths)
that already exist. */ that already exist. */

View file

@ -238,7 +238,6 @@ struct ClientSettings
} }
else if (trusted else if (trusted
|| name == settings.buildTimeout.name || name == settings.buildTimeout.name
|| name == settings.buildRepeat.name
|| name == settings.maxSilentTime.name || name == settings.maxSilentTime.name
|| name == settings.pollInterval.name || name == settings.pollInterval.name
|| name == "connect-timeout" || name == "connect-timeout"

View file

@ -147,7 +147,7 @@ void LocalStore::addTempRoot(const StorePath & path)
} catch (SysError & e) { } catch (SysError & e) {
/* The garbage collector may have exited, so we need to /* The garbage collector may have exited, so we need to
restart. */ restart. */
if (e.errNo == EPIPE) { if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
debug("GC socket disconnected"); debug("GC socket disconnected");
state->fdRootsSocket.close(); state->fdRootsSocket.close();
goto restart; goto restart;
@ -506,6 +506,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
Finally cleanup([&]() { Finally cleanup([&]() {
debug("GC roots server shutting down"); debug("GC roots server shutting down");
fdServer.close();
while (true) { while (true) {
auto item = remove_begin(*connections.lock()); auto item = remove_begin(*connections.lock());
if (!item) break; if (!item) break;

View file

@ -130,6 +130,10 @@ StringSet Settings::getDefaultSystemFeatures()
actually require anything special on the machines. */ actually require anything special on the machines. */
StringSet features{"nixos-test", "benchmark", "big-parallel"}; StringSet features{"nixos-test", "benchmark", "big-parallel"};
#if __linux__
features.insert("uid-range");
#endif
#if __linux__ #if __linux__
if (access("/dev/kvm", R_OK | W_OK) == 0) if (access("/dev/kvm", R_OK | W_OK) == 0)
features.insert("kvm"); features.insert("kvm");

View file

@ -46,6 +46,14 @@ struct PluginFilesSetting : public BaseSetting<Paths>
void set(const std::string & str, bool append = false) override; void set(const std::string & str, bool append = false) override;
}; };
const uint32_t maxIdsPerBuild =
#if __linux__
1 << 16
#else
1
#endif
;
class Settings : public Config { class Settings : public Config {
unsigned int getDefaultCores(); unsigned int getDefaultCores();
@ -275,6 +283,64 @@ public:
multi-user settings with untrusted users. multi-user settings with untrusted users.
)"}; )"};
Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
R"(
Whether to select UIDs for builds automatically, instead of using the
users in `build-users-group`.
UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = auto-allocate-uids
auto-allocate-uids = true
```
)"};
Setting<uint32_t> startId{this,
#if __linux__
0x34000000,
#else
56930,
#endif
"start-id",
"The first UID and GID to use for dynamic ID allocation."};
Setting<uint32_t> uidCount{this,
#if __linux__
maxIdsPerBuild * 128,
#else
128,
#endif
"id-count",
"The number of UIDs/GIDs to use for dynamic ID allocation."};
#if __linux__
Setting<bool> useCgroups{
this, false, "use-cgroups",
R"(
Whether to execute builds inside cgroups.
This is only supported on Linux.
Cgroups are required and enabled automatically for derivations
that require the `uid-range` system feature.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = cgroups
use-cgroups = true
```
)"};
#endif
Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26", Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
"Whether to impersonate a Linux 2.6 machine on newer kernels.", "Whether to impersonate a Linux 2.6 machine on newer kernels.",
{"build-impersonate-linux-26"}}; {"build-impersonate-linux-26"}};
@ -307,11 +373,6 @@ public:
)", )",
{"build-max-log-size"}}; {"build-max-log-size"}};
/* When buildRepeat > 0 and verboseBuild == true, whether to print
repeated builds (i.e. builds other than the first one) to
stderr. Hack to prevent Hydra logs from being polluted. */
bool printRepeatedBuilds = true;
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
"How often (in seconds) to poll for locks."}; "How often (in seconds) to poll for locks."};
@ -435,19 +496,6 @@ public:
Setting<bool> sandboxFallback{this, true, "sandbox-fallback", Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."}; "Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<size_t> buildRepeat{
this, 0, "repeat",
R"(
How many times to repeat builds to check whether they are
deterministic. The default value is 0. If the value is non-zero,
every build is repeated the specified number of times. If the
contents of any of the runs differs from the previous ones and
`enforce-determinism` is true, the build is rejected and the
resulting store paths are not registered as valid in Nixs
database.
)",
{"build-repeat"}};
#if __linux__ #if __linux__
Setting<std::string> sandboxShmSize{ Setting<std::string> sandboxShmSize{
this, "50%", "sandbox-dev-shm-size", this, "50%", "sandbox-dev-shm-size",
@ -511,10 +559,6 @@ public:
configuration file, and cannot be passed at the command line. configuration file, and cannot be passed at the command line.
)"}; )"};
Setting<bool> enforceDeterminism{
this, true, "enforce-determinism",
"Whether to fail if repeated builds produce different output. See `repeat`."};
Setting<Strings> trustedPublicKeys{ Setting<Strings> trustedPublicKeys{
this, this,
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
@ -563,10 +607,10 @@ public:
cache) must have a signature by a trusted key. A trusted key is one cache) must have a signature by a trusted key. A trusted key is one
listed in `trusted-public-keys`, or a public key counterpart to a listed in `trusted-public-keys`, or a public key counterpart to a
private key stored in a file listed in `secret-key-files`. private key stored in a file listed in `secret-key-files`.
Set to `false` to disable signature checking and trust all Set to `false` to disable signature checking and trust all
non-content-addressed paths unconditionally. non-content-addressed paths unconditionally.
(Content-addressed paths are inherently trustworthy and thus (Content-addressed paths are inherently trustworthy and thus
unaffected by this configuration option.) unaffected by this configuration option.)
)"}; )"};

View file

@ -255,8 +255,8 @@ private:
<< settings.maxLogSize; << settings.maxLogSize;
if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3)
conn.to conn.to
<< settings.buildRepeat << 0 // buildRepeat hasn't worked for ages anyway
<< settings.enforceDeterminism; << 0;
if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) {
conn.to << ((int) settings.keepFailed); conn.to << ((int) settings.keepFailed);

View file

@ -583,7 +583,10 @@ void canonicaliseTimestampAndPermissions(const Path & path)
} }
static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) static void canonicalisePathMetaData_(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{ {
checkInterrupt(); checkInterrupt();
@ -630,7 +633,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
However, ignore files that we chown'ed ourselves previously to However, ignore files that we chown'ed ourselves previously to
ensure that we don't fail on hard links within the same build ensure that we don't fail on hard links within the same build
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */ (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
throw BuildError("invalid ownership on file '%1%'", path); throw BuildError("invalid ownership on file '%1%'", path);
mode_t mode = st.st_mode & ~S_IFMT; mode_t mode = st.st_mode & ~S_IFMT;
@ -663,14 +666,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
DirEntries entries = readDirectory(path); DirEntries entries = readDirectory(path);
for (auto & i : entries) for (auto & i : entries)
canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
} }
} }
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{ {
canonicalisePathMetaData_(path, fromUid, inodesSeen); canonicalisePathMetaData_(path, uidRange, inodesSeen);
/* On platforms that don't have lchown(), the top-level path can't /* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */ be a symlink, since we can't change its ownership. */
@ -683,10 +689,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
} }
void canonicalisePathMetaData(const Path & path, uid_t fromUid) void canonicalisePathMetaData(const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange)
{ {
InodesSeen inodesSeen; InodesSeen inodesSeen;
canonicalisePathMetaData(path, fromUid, inodesSeen); canonicalisePathMetaData(path, uidRange, inodesSeen);
} }
@ -1331,7 +1338,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
autoGC(); autoGC();
canonicalisePathMetaData(realPath, -1); canonicalisePathMetaData(realPath, {});
optimisePath(realPath, repair); // FIXME: combine with hashPath() optimisePath(realPath, repair); // FIXME: combine with hashPath()
@ -1444,7 +1451,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
narHash = narSink.finish(); narHash = narSink.finish();
} }
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath
optimisePath(realPath, repair); optimisePath(realPath, repair);
@ -1486,7 +1493,7 @@ StorePath LocalStore::addTextToStore(
writeFile(realPath, s); writeFile(realPath, s);
canonicalisePathMetaData(realPath, -1); canonicalisePathMetaData(realPath, {});
StringSink sink; StringSink sink;
dumpString(s, sink); dumpString(s, sink);

View file

@ -310,9 +310,18 @@ typedef std::set<Inode> InodesSeen;
- the permissions are set of 444 or 555 (i.e., read-only with or - the permissions are set of 444 or 555 (i.e., read-only with or
without execute permission; setuid bits etc. are cleared) without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're - the owner and group are set to the Nix user and group, if we're
running as root. */ running as root.
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); If uidRange is not empty, this function will throw an error if it
void canonicalisePathMetaData(const Path & path, uid_t fromUid); encounters files owned by a user outside of the closed interval
[uidRange->first, uidRange->second].
*/
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen);
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange);
void canonicaliseTimestampAndPermissions(const Path & path); void canonicaliseTimestampAndPermissions(const Path & path);

View file

@ -2,105 +2,197 @@
#include "globals.hh" #include "globals.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include <grp.h>
#include <pwd.h> #include <pwd.h>
#include <grp.h>
#include <fcntl.h>
#include <unistd.h>
namespace nix { namespace nix {
UserLock::UserLock() struct SimpleUserLock : UserLock
{ {
assert(settings.buildUsersGroup != ""); AutoCloseFD fdUserLock;
createDirs(settings.nixStateDir + "/userpool"); uid_t uid;
} gid_t gid;
std::vector<gid_t> supplementaryGIDs;
bool UserLock::findFreeUser() { uid_t getUID() override { assert(uid); return uid; }
if (enabled()) return true; uid_t getUIDCount() override { return 1; }
gid_t getGID() override { assert(gid); return gid; }
/* Get the members of the build-users-group. */ std::vector<gid_t> getSupplementaryGIDs() override { return supplementaryGIDs; }
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
throw Error("the group '%1%' specified in 'build-users-group' does not exist",
settings.buildUsersGroup);
gid = gr->gr_gid;
/* Copy the result of getgrnam. */ static std::unique_ptr<UserLock> acquire()
Strings users; {
for (char * * p = gr->gr_mem; *p; ++p) { assert(settings.buildUsersGroup != "");
debug("found build user '%1%'", *p); createDirs(settings.nixStateDir + "/userpool");
users.push_back(*p);
}
if (users.empty()) /* Get the members of the build-users-group. */
throw Error("the build users group '%1%' has no members", struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
settings.buildUsersGroup); if (!gr)
throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
/* Find a user account that isn't currently in use for another /* Copy the result of getgrnam. */
build. */ Strings users;
for (auto & i : users) { for (char * * p = gr->gr_mem; *p; ++p) {
debug("trying user '%1%'", i); debug("found build user '%s'", *p);
users.push_back(*p);
struct passwd * pw = getpwnam(i.c_str());
if (!pw)
throw Error("the user '%1%' in the group '%2%' does not exist",
i, settings.buildUsersGroup);
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock '%1%'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
fdUserLock = std::move(fd);
user = i;
uid = pw->pw_uid;
/* Sanity check... */
if (uid == getuid() || uid == geteuid())
throw Error("the Nix user should not be a member of '%1%'",
settings.buildUsersGroup);
#if __linux__
/* Get the list of supplementary groups of this build user. This
is usually either empty or contains a group such as "kvm". */
int ngroups = 32; // arbitrary initial guess
supplementaryGIDs.resize(ngroups);
int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(),
&ngroups);
// Our initial size of 32 wasn't sufficient, the correct size has
// been stored in ngroups, so we try again.
if (err == -1) {
supplementaryGIDs.resize(ngroups);
err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(),
&ngroups);
}
// If it failed once more, then something must be broken.
if (err == -1)
throw Error("failed to get list of supplementary groups for '%1%'",
pw->pw_name);
// Finally, trim back the GID list to its real size
supplementaryGIDs.resize(ngroups);
#endif
isEnabled = true;
return true;
} }
if (users.empty())
throw Error("the build users group '%s' has no members", settings.buildUsersGroup);
/* Find a user account that isn't currently in use for another
build. */
for (auto & i : users) {
debug("trying user '%s'", i);
struct passwd * pw = getpwnam(i.c_str());
if (!pw)
throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup);
auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid);
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock '%s'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
auto lock = std::make_unique<SimpleUserLock>();
lock->fdUserLock = std::move(fd);
lock->uid = pw->pw_uid;
lock->gid = gr->gr_gid;
/* Sanity check... */
if (lock->uid == getuid() || lock->uid == geteuid())
throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup);
#if __linux__
/* Get the list of supplementary groups of this build
user. This is usually either empty or contains a
group such as "kvm". */
int ngroups = 32; // arbitrary initial guess
std::vector<gid_t> gids;
gids.resize(ngroups);
int err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
/* Our initial size of 32 wasn't sufficient, the
correct size has been stored in ngroups, so we try
again. */
if (err == -1) {
gids.resize(ngroups);
err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
}
// If it failed once more, then something must be broken.
if (err == -1)
throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name);
// Finally, trim back the GID list to its real size.
for (auto i = 0; i < ngroups; i++)
if (gids[i] != lock->gid)
lock->supplementaryGIDs.push_back(gids[i]);
#endif
return lock;
}
}
return nullptr;
} }
};
return false; struct AutoUserLock : UserLock
}
void UserLock::kill()
{ {
killUser(uid); AutoCloseFD fdUserLock;
uid_t firstUid = 0;
gid_t firstGid = 0;
uid_t nrIds = 1;
uid_t getUID() override { assert(firstUid); return firstUid; }
gid_t getUIDCount() override { return nrIds; }
gid_t getGID() override { assert(firstGid); return firstGid; }
std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot)
{
settings.requireExperimentalFeature(Xp::AutoAllocateUids);
assert(settings.startId > 0);
assert(settings.uidCount % maxIdsPerBuild == 0);
assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
assert(nrIds <= maxIdsPerBuild);
createDirs(settings.nixStateDir + "/userpool2");
size_t nrSlots = settings.uidCount / maxIdsPerBuild;
for (size_t i = 0; i < nrSlots; i++) {
debug("trying user slot '%d'", i);
createDirs(settings.nixStateDir + "/userpool2");
auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i);
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock '%s'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
auto firstUid = settings.startId + i * maxIdsPerBuild;
auto pw = getpwuid(firstUid);
if (pw)
throw Error("auto-allocated UID %d clashes with existing user account '%s'", firstUid, pw->pw_name);
auto lock = std::make_unique<AutoUserLock>();
lock->fdUserLock = std::move(fd);
lock->firstUid = firstUid;
if (useChroot)
lock->firstGid = firstUid;
else {
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
lock->firstGid = gr->gr_gid;
}
lock->nrIds = nrIds;
return lock;
}
}
return nullptr;
}
};
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
{
if (settings.autoAllocateUids)
return AutoUserLock::acquire(nrIds, useChroot);
else
return SimpleUserLock::acquire();
}
bool useBuildUsers()
{
#if __linux__
static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
return b;
#elif __APPLE__
static bool b = settings.buildUsersGroup != "" && getuid() == 0;
return b;
#else
return false;
#endif
} }
} }

View file

@ -1,37 +1,38 @@
#pragma once #pragma once
#include "sync.hh"
#include "types.hh" #include "types.hh"
#include "util.hh"
#include <optional>
#include <sys/types.h>
namespace nix { namespace nix {
class UserLock struct UserLock
{ {
private: virtual ~UserLock() { }
Path fnUserLock;
AutoCloseFD fdUserLock;
bool isEnabled = false; /* Get the first and last UID. */
std::string user; std::pair<uid_t, uid_t> getUIDRange()
uid_t uid = 0; {
gid_t gid = 0; auto first = getUID();
std::vector<gid_t> supplementaryGIDs; return {first, first + getUIDCount() - 1};
}
public: /* Get the first UID. */
UserLock(); virtual uid_t getUID() = 0;
void kill(); virtual uid_t getUIDCount() = 0;
std::string getUser() { return user; } virtual gid_t getGID() = 0;
uid_t getUID() { assert(uid); return uid; }
uid_t getGID() { assert(gid); return gid; }
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
bool findFreeUser();
bool enabled() { return isEnabled; }
virtual std::vector<gid_t> getSupplementaryGIDs() = 0;
}; };
/* Acquire a user lock for a UID range of size `nrIds`. Note that this
may return nullptr if no user is available. */
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot);
bool useBuildUsers();
} }

View file

@ -1,6 +1,5 @@
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "archive.hh" #include "archive.hh"
#include "json.hh"
#include <map> #include <map>
#include <stack> #include <stack>
@ -243,42 +242,43 @@ ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
return make_ref<NarAccessor>(listing, getNarBytes); return make_ref<NarAccessor>(listing, getNarBytes);
} }
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, using nlohmann::json;
const Path & path, bool recurse) json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
{ {
auto st = accessor->stat(path); auto st = accessor->stat(path);
auto obj = res.object(); json obj = json::object();
switch (st.type) { switch (st.type) {
case FSAccessor::Type::tRegular: case FSAccessor::Type::tRegular:
obj.attr("type", "regular"); obj["type"] = "regular";
obj.attr("size", st.fileSize); obj["size"] = st.fileSize;
if (st.isExecutable) if (st.isExecutable)
obj.attr("executable", true); obj["executable"] = true;
if (st.narOffset) if (st.narOffset)
obj.attr("narOffset", st.narOffset); obj["narOffset"] = st.narOffset;
break; break;
case FSAccessor::Type::tDirectory: case FSAccessor::Type::tDirectory:
obj.attr("type", "directory"); obj["type"] = "directory";
{ {
auto res2 = obj.object("entries"); obj["entries"] = json::object();
json &res2 = obj["entries"];
for (auto & name : accessor->readDirectory(path)) { for (auto & name : accessor->readDirectory(path)) {
if (recurse) { if (recurse) {
auto res3 = res2.placeholder(name); res2[name] = listNar(accessor, path + "/" + name, true);
listNar(res3, accessor, path + "/" + name, true);
} else } else
res2.object(name); res2[name] = json::object();
} }
} }
break; break;
case FSAccessor::Type::tSymlink: case FSAccessor::Type::tSymlink:
obj.attr("type", "symlink"); obj["type"] = "symlink";
obj.attr("target", accessor->readLink(path)); obj["target"] = accessor->readLink(path);
break; break;
default: default:
throw Error("path '%s' does not exist in NAR", path); throw Error("path '%s' does not exist in NAR", path);
} }
return obj;
} }
} }

View file

@ -2,6 +2,7 @@
#include <functional> #include <functional>
#include <nlohmann/json_fwd.hpp>
#include "fs-accessor.hh" #include "fs-accessor.hh"
namespace nix { namespace nix {
@ -24,11 +25,8 @@ ref<FSAccessor> makeLazyNarAccessor(
const std::string & listing, const std::string & listing,
GetNarBytes getNarBytes); GetNarBytes getNarBytes);
class JSONPlaceholder;
/* Write a JSON representation of the contents of a NAR (except file /* Write a JSON representation of the contents of a NAR (except file
contents). */ contents). */
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
const Path & path, bool recurse);
} }

View file

@ -2,7 +2,6 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex> #include <regex>
#include "json.hh"
namespace nix { namespace nix {
@ -90,6 +89,7 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet ParsedDerivation::getRequiredSystemFeatures() const
{ {
// FIXME: cache this?
StringSet res; StringSet res;
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i); res.insert(i);
@ -125,6 +125,11 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true); return getBoolAttr("allowSubstitutes", true);
} }
bool ParsedDerivation::useUidRange() const
{
return getRequiredSystemFeatures().count("uid-range");
}
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
@ -144,16 +149,11 @@ std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & s
auto e = json.find("exportReferencesGraph"); auto e = json.find("exportReferencesGraph");
if (e != json.end() && e->is_object()) { if (e != json.end() && e->is_object()) {
for (auto i = e->begin(); i != e->end(); ++i) { for (auto i = e->begin(); i != e->end(); ++i) {
std::ostringstream str; StorePathSet storePaths;
{ for (auto & p : *i)
JSONPlaceholder jsonRoot(str, true); storePaths.insert(store.parseStorePath(p.get<std::string>()));
StorePathSet storePaths; json[i.key()] = store.pathInfoToJSON(
for (auto & p : *i) store.exportReferences(storePaths, inputPaths), false, true);
storePaths.insert(store.parseStorePath(p.get<std::string>()));
store.pathInfoToJSON(jsonRoot,
store.exportReferences(storePaths, inputPaths), false, true);
}
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
} }
} }

View file

@ -38,6 +38,8 @@ public:
bool substitutesAllowed() const; bool substitutesAllowed() const;
bool useUidRange() const;
std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
}; };

View file

@ -1,6 +1,6 @@
#include <nlohmann/json.hpp>
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "json.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@ -38,10 +38,8 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
if (cacheDir != "") { if (cacheDir != "") {
try { try {
std::ostringstream str; nlohmann::json j = listNar(narAccessor, "", true);
JSONPlaceholder jsonRoot(str); writeFile(makeCacheFile(hashPart, "ls"), j.dump());
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(hashPart, "ls"), str.str());
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }

View file

@ -6,14 +6,16 @@
#include "util.hh" #include "util.hh"
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh"
#include "url.hh" #include "url.hh"
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include <nlohmann/json.hpp>
#include <regex> #include <regex>
using json = nlohmann::json;
namespace nix { namespace nix {
@ -838,56 +840,53 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
return paths; return paths;
} }
json Store::pathInfoToJSON(const StorePathSet & storePaths,
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, bool includeImpureInfo, bool showClosureSize,
Base hashBase, Base hashBase,
AllowInvalidFlag allowInvalid) AllowInvalidFlag allowInvalid)
{ {
auto jsonList = jsonOut.list(); json::array_t jsonList = json::array();
for (auto & storePath : storePaths) { for (auto & storePath : storePaths) {
auto jsonPath = jsonList.object(); auto& jsonPath = jsonList.emplace_back(json::object());
try { try {
auto info = queryPathInfo(storePath); auto info = queryPathInfo(storePath);
jsonPath.attr("path", printStorePath(info->path)); jsonPath["path"] = printStorePath(info->path);
jsonPath jsonPath["narHash"] = info->narHash.to_string(hashBase, true);
.attr("narHash", info->narHash.to_string(hashBase, true)) jsonPath["narSize"] = info->narSize;
.attr("narSize", info->narSize);
{ {
auto jsonRefs = jsonPath.list("references"); auto& jsonRefs = (jsonPath["references"] = json::array());
for (auto & ref : info->references) for (auto & ref : info->references)
jsonRefs.elem(printStorePath(ref)); jsonRefs.emplace_back(printStorePath(ref));
} }
if (info->ca) if (info->ca)
jsonPath.attr("ca", renderContentAddress(info->ca)); jsonPath["ca"] = renderContentAddress(info->ca);
std::pair<uint64_t, uint64_t> closureSizes; std::pair<uint64_t, uint64_t> closureSizes;
if (showClosureSize) { if (showClosureSize) {
closureSizes = getClosureSize(info->path); closureSizes = getClosureSize(info->path);
jsonPath.attr("closureSize", closureSizes.first); jsonPath["closureSize"] = closureSizes.first;
} }
if (includeImpureInfo) { if (includeImpureInfo) {
if (info->deriver) if (info->deriver)
jsonPath.attr("deriver", printStorePath(*info->deriver)); jsonPath["deriver"] = printStorePath(*info->deriver);
if (info->registrationTime) if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime); jsonPath["registrationTime"] = info->registrationTime;
if (info->ultimate) if (info->ultimate)
jsonPath.attr("ultimate", info->ultimate); jsonPath["ultimate"] = info->ultimate;
if (!info->sigs.empty()) { if (!info->sigs.empty()) {
auto jsonSigs = jsonPath.list("signatures");
for (auto & sig : info->sigs) for (auto & sig : info->sigs)
jsonSigs.elem(sig); jsonPath["signatures"].push_back(sig);
} }
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
@ -895,21 +894,22 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (narInfo) { if (narInfo) {
if (!narInfo->url.empty()) if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url); jsonPath["url"] = narInfo->url;
if (narInfo->fileHash) if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true);
if (narInfo->fileSize) if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize); jsonPath["downloadSize"] = narInfo->fileSize;
if (showClosureSize) if (showClosureSize)
jsonPath.attr("closureDownloadSize", closureSizes.second); jsonPath["closureDownloadSize"] = closureSizes.second;
} }
} }
} catch (InvalidPath &) { } catch (InvalidPath &) {
jsonPath.attr("path", printStorePath(storePath)); jsonPath["path"] = printStorePath(storePath);
jsonPath.attr("valid", false); jsonPath["valid"] = false;
} }
} }
return jsonList;
} }

View file

@ -14,6 +14,7 @@
#include "path-info.hh" #include "path-info.hh"
#include "repair-flag.hh" #include "repair-flag.hh"
#include <nlohmann/json_fwd.hpp>
#include <atomic> #include <atomic>
#include <limits> #include <limits>
#include <map> #include <map>
@ -68,7 +69,6 @@ struct Derivation;
class FSAccessor; class FSAccessor;
class NarInfoDiskCache; class NarInfoDiskCache;
class Store; class Store;
class JSONPlaceholder;
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
@ -512,7 +512,7 @@ public:
variable elements such as the registration time are variable elements such as the registration time are
included. If showClosureSize is true, the closure size of included. If showClosureSize is true, the closure size of
each path is included. */ each path is included. */
void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, nlohmann::json pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, bool includeImpureInfo, bool showClosureSize,
Base hashBase = Base32, Base hashBase = Base32,
AllowInvalidFlag allowInvalid = DisallowInvalid); AllowInvalidFlag allowInvalid = DisallowInvalid);

148
src/libutil/cgroup.cc Normal file
View file

@ -0,0 +1,148 @@
#if __linux__
#include "cgroup.hh"
#include "util.hh"
#include "finally.hh"
#include <chrono>
#include <cmath>
#include <regex>
#include <unordered_set>
#include <thread>
#include <dirent.h>
#include <mntent.h>
namespace nix {
std::optional<Path> getCgroupFS()
{
static auto res = [&]() -> std::optional<Path> {
auto fp = fopen("/proc/mounts", "r");
if (!fp) return std::nullopt;
Finally delFP = [&]() { fclose(fp); };
while (auto ent = getmntent(fp))
if (std::string_view(ent->mnt_type) == "cgroup2")
return ent->mnt_dir;
return std::nullopt;
}();
return res;
}
// FIXME: obsolete, check for cgroup2
std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
{
std::map<std::string, std::string> cgroups;
for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cgroupFile), "\n")) {
static std::regex regex("([0-9]+):([^:]*):(.*)");
std::smatch match;
if (!std::regex_match(line, match, regex))
throw Error("invalid line '%s' in '%s'", line, cgroupFile);
std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2];
cgroups.insert_or_assign(name, match[3]);
}
return cgroups;
}
static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
{
if (!pathExists(cgroup)) return {};
auto procsFile = cgroup + "/cgroup.procs";
if (!pathExists(procsFile))
throw Error("'%s' is not a cgroup", cgroup);
/* Use the fast way to kill every process in a cgroup, if
available. */
auto killFile = cgroup + "/cgroup.kill";
if (pathExists(killFile))
writeFile(killFile, "1");
/* Otherwise, manually kill every process in the subcgroups and
this cgroup. */
for (auto & entry : readDirectory(cgroup)) {
if (entry.type != DT_DIR) continue;
destroyCgroup(cgroup + "/" + entry.name, false);
}
int round = 1;
std::unordered_set<pid_t> pidsShown;
while (true) {
auto pids = tokenizeString<std::vector<std::string>>(readFile(procsFile));
if (pids.empty()) break;
if (round > 20)
throw Error("cannot kill cgroup '%s'", cgroup);
for (auto & pid_s : pids) {
pid_t pid;
if (auto o = string2Int<pid_t>(pid_s))
pid = *o;
else
throw Error("invalid pid '%s'", pid);
if (pidsShown.insert(pid).second) {
try {
auto cmdline = readFile(fmt("/proc/%d/cmdline", pid));
using namespace std::string_literals;
warn("killing stray builder process %d (%s)...",
pid, trim(replaceStrings(cmdline, "\0"s, " ")));
} catch (SysError &) {
}
}
// FIXME: pid wraparound
if (kill(pid, SIGKILL) == -1 && errno != ESRCH)
throw SysError("killing member %d of cgroup '%s'", pid, cgroup);
}
auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10)));
if (sleep.count() > 100)
printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup);
std::this_thread::sleep_for(sleep);
round++;
}
CgroupStats stats;
if (returnStats) {
auto cpustatPath = cgroup + "/cpu.stat";
if (pathExists(cpustatPath)) {
for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cpustatPath), "\n")) {
std::string_view userPrefix = "user_usec ";
if (hasPrefix(line, userPrefix)) {
auto n = string2Int<uint64_t>(line.substr(userPrefix.size()));
if (n) stats.cpuUser = std::chrono::microseconds(*n);
}
std::string_view systemPrefix = "system_usec ";
if (hasPrefix(line, systemPrefix)) {
auto n = string2Int<uint64_t>(line.substr(systemPrefix.size()));
if (n) stats.cpuSystem = std::chrono::microseconds(*n);
}
}
}
}
if (rmdir(cgroup.c_str()) == -1)
throw SysError("deleting cgroup '%s'", cgroup);
return stats;
}
CgroupStats destroyCgroup(const Path & cgroup)
{
return destroyCgroup(cgroup, true);
}
}
#endif

29
src/libutil/cgroup.hh Normal file
View file

@ -0,0 +1,29 @@
#pragma once
#if __linux__
#include <chrono>
#include <optional>
#include "types.hh"
namespace nix {
std::optional<Path> getCgroupFS();
std::map<std::string, std::string> getCgroups(const Path & cgroupFile);
struct CgroupStats
{
std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
};
/* Destroy the cgroup denoted by 'path'. The postcondition is that
'path' does not exist, and thus any processes in the cgroup have
been killed. Also return statistics from the cgroup just before
destruction. */
CgroupStats destroyCgroup(const Path & cgroup);
}
#endif

View file

@ -14,6 +14,8 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::NoUrlLiterals, "no-url-literals" }, { Xp::NoUrlLiterals, "no-url-literals" },
{ Xp::FetchClosure, "fetch-closure" }, { Xp::FetchClosure, "fetch-closure" },
{ Xp::ReplFlake, "repl-flake" }, { Xp::ReplFlake, "repl-flake" },
{ Xp::AutoAllocateUids, "auto-allocate-uids" },
{ Xp::Cgroups, "cgroups" },
}; };
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

View file

@ -23,6 +23,8 @@ enum struct ExperimentalFeature
NoUrlLiterals, NoUrlLiterals,
FetchClosure, FetchClosure,
ReplFlake, ReplFlake,
AutoAllocateUids,
Cgroups,
}; };
/** /**

View file

@ -1,5 +1,6 @@
#include <sys/time.h> #include <sys/time.h>
#include <filesystem> #include <filesystem>
#include <atomic>
#include "finally.hh" #include "finally.hh"
#include "util.hh" #include "util.hh"
@ -10,7 +11,7 @@ namespace fs = std::filesystem;
namespace nix { namespace nix {
static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
int & counter) std::atomic<unsigned int> & counter)
{ {
tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
if (includePid) if (includePid)
@ -22,9 +23,9 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
Path createTempDir(const Path & tmpRoot, const Path & prefix, Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter, mode_t mode) bool includePid, bool useGlobalCounter, mode_t mode)
{ {
static int globalCounter = 0; static std::atomic<unsigned int> globalCounter = 0;
int localCounter = 0; std::atomic<unsigned int> localCounter = 0;
int & counter(useGlobalCounter ? globalCounter : localCounter); auto & counter(useGlobalCounter ? globalCounter : localCounter);
while (1) { while (1) {
checkInterrupt(); checkInterrupt();

View file

@ -1,203 +0,0 @@
#include "json.hh"
#include <iomanip>
#include <cstdint>
#include <cstring>
namespace nix {
template<>
void toJSON<std::string_view>(std::ostream & str, const std::string_view & s)
{
constexpr size_t BUF_SIZE = 4096;
char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
size_t bufPos = 0;
const auto flush = [&] {
str.write(buf, bufPos);
bufPos = 0;
};
const auto put = [&] (char c) {
buf[bufPos++] = c;
};
put('"');
for (auto i = s.begin(); i != s.end(); i++) {
if (bufPos >= BUF_SIZE) flush();
if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
else if (*i == '\n') { put('\\'); put('n'); }
else if (*i == '\r') { put('\\'); put('r'); }
else if (*i == '\t') { put('\\'); put('t'); }
else if (*i >= 0 && *i < 32) {
const char hex[17] = "0123456789abcdef";
put('\\');
put('u');
put(hex[(uint16_t(*i) >> 12) & 0xf]);
put(hex[(uint16_t(*i) >> 8) & 0xf]);
put(hex[(uint16_t(*i) >> 4) & 0xf]);
put(hex[(uint16_t(*i) >> 0) & 0xf]);
}
else put(*i);
}
put('"');
flush();
}
void toJSON(std::ostream & str, const char * s)
{
if (!s) str << "null"; else toJSON(str, std::string_view(s));
}
template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; }
template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; }
template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; }
template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); }
template<> void toJSON<bool>(std::ostream & str, const bool & b)
{
str << (b ? "true" : "false");
}
template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b)
{
str << "null";
}
JSONWriter::JSONWriter(std::ostream & str, bool indent)
: state(new JSONState(str, indent))
{
state->stack++;
}
JSONWriter::JSONWriter(JSONState * state)
: state(state)
{
state->stack++;
}
JSONWriter::~JSONWriter()
{
if (state) {
assertActive();
state->stack--;
if (state->stack == 0) delete state;
}
}
void JSONWriter::comma()
{
assertActive();
if (first) {
first = false;
} else {
state->str << ',';
}
if (state->indent) indent();
}
void JSONWriter::indent()
{
state->str << '\n' << std::string(state->depth * 2, ' ');
}
void JSONList::open()
{
state->depth++;
state->str << '[';
}
JSONList::~JSONList()
{
state->depth--;
if (state->indent && !first) indent();
state->str << "]";
}
JSONList JSONList::list()
{
comma();
return JSONList(state);
}
JSONObject JSONList::object()
{
comma();
return JSONObject(state);
}
JSONPlaceholder JSONList::placeholder()
{
comma();
return JSONPlaceholder(state);
}
void JSONObject::open()
{
state->depth++;
state->str << '{';
}
JSONObject::~JSONObject()
{
if (state) {
state->depth--;
if (state->indent && !first) indent();
state->str << "}";
}
}
void JSONObject::attr(std::string_view s)
{
comma();
toJSON(state->str, s);
state->str << ':';
if (state->indent) state->str << ' ';
}
JSONList JSONObject::list(std::string_view name)
{
attr(name);
return JSONList(state);
}
JSONObject JSONObject::object(std::string_view name)
{
attr(name);
return JSONObject(state);
}
JSONPlaceholder JSONObject::placeholder(std::string_view name)
{
attr(name);
return JSONPlaceholder(state);
}
JSONList JSONPlaceholder::list()
{
assertValid();
first = false;
return JSONList(state);
}
JSONObject JSONPlaceholder::object()
{
assertValid();
first = false;
return JSONObject(state);
}
JSONPlaceholder::~JSONPlaceholder()
{
if (first) {
assert(std::uncaught_exceptions());
if (state->stack != 0)
write(nullptr);
}
}
}

View file

@ -1,185 +0,0 @@
#pragma once
#include <iostream>
#include <vector>
#include <cassert>
namespace nix {
void toJSON(std::ostream & str, const char * s);
template<typename T>
void toJSON(std::ostream & str, const T & n);
class JSONWriter
{
protected:
struct JSONState
{
std::ostream & str;
bool indent;
size_t depth = 0;
size_t stack = 0;
JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
~JSONState()
{
assert(stack == 0);
}
};
JSONState * state;
bool first = true;
JSONWriter(std::ostream & str, bool indent);
JSONWriter(JSONState * state);
~JSONWriter();
void assertActive()
{
assert(state->stack != 0);
}
void comma();
void indent();
};
class JSONObject;
class JSONPlaceholder;
class JSONList : JSONWriter
{
private:
friend class JSONObject;
friend class JSONPlaceholder;
void open();
JSONList(JSONState * state)
: JSONWriter(state)
{
open();
}
public:
JSONList(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
open();
}
~JSONList();
template<typename T>
JSONList & elem(const T & v)
{
comma();
toJSON(state->str, v);
return *this;
}
JSONList list();
JSONObject object();
JSONPlaceholder placeholder();
};
class JSONObject : JSONWriter
{
private:
friend class JSONList;
friend class JSONPlaceholder;
void open();
JSONObject(JSONState * state)
: JSONWriter(state)
{
open();
}
void attr(std::string_view s);
public:
JSONObject(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
open();
}
JSONObject(const JSONObject & obj) = delete;
JSONObject(JSONObject && obj)
: JSONWriter(obj.state)
{
obj.state = 0;
}
~JSONObject();
template<typename T>
JSONObject & attr(std::string_view name, const T & v)
{
attr(name);
toJSON(state->str, v);
return *this;
}
JSONList list(std::string_view name);
JSONObject object(std::string_view name);
JSONPlaceholder placeholder(std::string_view name);
};
class JSONPlaceholder : JSONWriter
{
private:
friend class JSONList;
friend class JSONObject;
JSONPlaceholder(JSONState * state)
: JSONWriter(state)
{
}
void assertValid()
{
assertActive();
assert(first);
}
public:
JSONPlaceholder(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
}
~JSONPlaceholder();
template<typename T>
void write(const T & v)
{
assertValid();
first = false;
toJSON(state->str, v);
}
JSONList list();
JSONObject object();
};
}

View file

@ -1,193 +0,0 @@
#include "json.hh"
#include <gtest/gtest.h>
#include <sstream>
namespace nix {
/* ----------------------------------------------------------------------------
* toJSON
* --------------------------------------------------------------------------*/
TEST(toJSON, quotesCharPtr) {
const char* input = "test";
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "\"test\"");
}
TEST(toJSON, quotesStdString) {
std::string input = "test";
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "\"test\"");
}
TEST(toJSON, convertsNullptrtoNull) {
auto input = nullptr;
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "null");
}
TEST(toJSON, convertsNullToNull) {
const char* input = 0;
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "null");
}
TEST(toJSON, convertsFloat) {
auto input = 1.024f;
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "1.024");
}
TEST(toJSON, convertsDouble) {
const double input = 1.024;
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "1.024");
}
TEST(toJSON, convertsBool) {
auto input = false;
std::stringstream out;
toJSON(out, input);
ASSERT_EQ(out.str(), "false");
}
TEST(toJSON, quotesTab) {
std::stringstream out;
toJSON(out, "\t");
ASSERT_EQ(out.str(), "\"\\t\"");
}
TEST(toJSON, quotesNewline) {
std::stringstream out;
toJSON(out, "\n");
ASSERT_EQ(out.str(), "\"\\n\"");
}
TEST(toJSON, quotesCreturn) {
std::stringstream out;
toJSON(out, "\r");
ASSERT_EQ(out.str(), "\"\\r\"");
}
TEST(toJSON, quotesCreturnNewLine) {
std::stringstream out;
toJSON(out, "\r\n");
ASSERT_EQ(out.str(), "\"\\r\\n\"");
}
TEST(toJSON, quotesDoublequotes) {
std::stringstream out;
toJSON(out, "\"");
ASSERT_EQ(out.str(), "\"\\\"\"");
}
TEST(toJSON, substringEscape) {
std::stringstream out;
std::string_view s = "foo\t";
toJSON(out, s.substr(3));
ASSERT_EQ(out.str(), "\"\\t\"");
}
/* ----------------------------------------------------------------------------
* JSONObject
* --------------------------------------------------------------------------*/
TEST(JSONObject, emptyObject) {
std::stringstream out;
{
JSONObject t(out);
}
ASSERT_EQ(out.str(), "{}");
}
TEST(JSONObject, objectWithList) {
std::stringstream out;
{
JSONObject t(out);
auto l = t.list("list");
l.elem("element");
}
ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
}
TEST(JSONObject, objectWithListIndent) {
std::stringstream out;
{
JSONObject t(out, true);
auto l = t.list("list");
l.elem("element");
}
ASSERT_EQ(out.str(),
R"#({
"list": [
"element"
]
})#");
}
TEST(JSONObject, objectWithPlaceholderAndList) {
std::stringstream out;
{
JSONObject t(out);
auto l = t.placeholder("list");
l.list().elem("element");
}
ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
}
TEST(JSONObject, objectWithPlaceholderAndObject) {
std::stringstream out;
{
JSONObject t(out);
auto l = t.placeholder("object");
l.object().attr("key", "value");
}
ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#");
}
/* ----------------------------------------------------------------------------
* JSONList
* --------------------------------------------------------------------------*/
TEST(JSONList, empty) {
std::stringstream out;
{
JSONList l(out);
}
ASSERT_EQ(out.str(), R"#([])#");
}
TEST(JSONList, withElements) {
std::stringstream out;
{
JSONList l(out);
l.elem("one");
l.object();
l.placeholder().write("three");
}
ASSERT_EQ(out.str(), R"#(["one",{},"three"])#");
}
}

View file

@ -2,6 +2,7 @@
#include "sync.hh" #include "sync.hh"
#include "finally.hh" #include "finally.hh"
#include "serialise.hh" #include "serialise.hh"
#include "cgroup.hh"
#include <array> #include <array>
#include <cctype> #include <cctype>
@ -36,7 +37,6 @@
#include <sys/prctl.h> #include <sys/prctl.h>
#include <sys/resource.h> #include <sys/resource.h>
#include <mntent.h>
#include <cmath> #include <cmath>
#endif #endif
@ -727,45 +727,22 @@ unsigned int getMaxCPU()
{ {
#if __linux__ #if __linux__
try { try {
FILE *fp = fopen("/proc/mounts", "r"); auto cgroupFS = getCgroupFS();
if (!fp) if (!cgroupFS) return 0;
return 0;
Strings cgPathParts; auto cgroups = getCgroups("/proc/self/cgroup");
auto cgroup = cgroups[""];
if (cgroup == "") return 0;
struct mntent *ent; auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
while ((ent = getmntent(fp))) {
std::string mountType, mountPath;
mountType = ent->mnt_type; auto cpuMax = readFile(cpuFile);
mountPath = ent->mnt_dir; auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
auto quota = cpuMaxParts[0];
if (mountType == "cgroup2") { auto period = cpuMaxParts[1];
cgPathParts.push_back(mountPath); if (quota != "max")
break; return std::ceil(std::stoi(quota) / std::stof(period));
} } catch (Error &) { ignoreException(lvlDebug); }
}
fclose(fp);
if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) {
std::string currentCgroup = readFile("/proc/self/cgroup");
Strings cgValues = tokenizeString<Strings>(currentCgroup, ":");
cgPathParts.push_back(trim(cgValues.back(), "\n"));
cgPathParts.push_back("cpu.max");
std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts));
if (pathExists(fullCgPath)) {
std::string cpuMax = readFile(fullCgPath);
std::vector<std::string> cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " ");
std::string quota = cpuMaxParts[0];
std::string period = trim(cpuMaxParts[1], "\n");
if (quota != "max")
return std::ceil(std::stoi(quota) / std::stof(period));
}
}
} catch (Error &) { ignoreException(); }
#endif #endif
return 0; return 0;
@ -1427,7 +1404,7 @@ std::string shellEscape(const std::string_view s)
} }
void ignoreException() void ignoreException(Verbosity lvl)
{ {
/* Make sure no exceptions leave this function. /* Make sure no exceptions leave this function.
printError() also throws when remote is closed. */ printError() also throws when remote is closed. */
@ -1435,7 +1412,7 @@ void ignoreException()
try { try {
throw; throw;
} catch (std::exception & e) { } catch (std::exception & e) {
printError("error (ignored): %1%", e.what()); printMsg(lvl, "error (ignored): %1%", e.what());
} }
} catch (...) { } } catch (...) { }
} }

View file

@ -528,7 +528,7 @@ std::string shellEscape(const std::string_view s);
/* Exception handling in destructors: print an error message, then /* Exception handling in destructors: print an error message, then
ignore the exception. */ ignore the exception. */
void ignoreException(); void ignoreException(Verbosity lvl = lvlError);

View file

@ -12,7 +12,6 @@
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "user-env.hh" #include "user-env.hh"
#include "util.hh" #include "util.hh"
#include "json.hh"
#include "value-to-json.hh" #include "value-to-json.hh"
#include "xml-writer.hh" #include "xml-writer.hh"
#include "legacy.hh" #include "legacy.hh"
@ -26,6 +25,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <unistd.h> #include <unistd.h>
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
using std::cout; using std::cout;
@ -911,43 +911,47 @@ static VersionDiff compareVersionAgainstSet(
static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta) static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta)
{ {
JSONObject topObj(cout, true); using nlohmann::json;
json topObj = json::object();
for (auto & i : elems) { for (auto & i : elems) {
try { try {
if (i.hasFailed()) continue; if (i.hasFailed()) continue;
JSONObject pkgObj = topObj.object(i.attrPath);
auto drvName = DrvName(i.queryName()); auto drvName = DrvName(i.queryName());
pkgObj.attr("name", drvName.fullName); json &pkgObj = topObj[i.attrPath];
pkgObj.attr("pname", drvName.name); pkgObj = {
pkgObj.attr("version", drvName.version); {"name", drvName.fullName},
pkgObj.attr("system", i.querySystem()); {"pname", drvName.name},
pkgObj.attr("outputName", i.queryOutputName()); {"version", drvName.version},
{"system", i.querySystem()},
{"outputName", i.queryOutputName()},
};
{ {
DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
JSONObject outputObj = pkgObj.object("outputs"); json &outputObj = pkgObj["outputs"];
outputObj = json::object();
for (auto & j : outputs) { for (auto & j : outputs) {
if (j.second) if (j.second)
outputObj.attr(j.first, globals.state->store->printStorePath(*j.second)); outputObj[j.first] = globals.state->store->printStorePath(*j.second);
else else
outputObj.attr(j.first, nullptr); outputObj[j.first] = nullptr;
} }
} }
if (printMeta) { if (printMeta) {
JSONObject metaObj = pkgObj.object("meta"); json &metaObj = pkgObj["meta"];
metaObj = json::object();
StringSet metaNames = i.queryMetaNames(); StringSet metaNames = i.queryMetaNames();
for (auto & j : metaNames) { for (auto & j : metaNames) {
Value * v = i.queryMeta(j); Value * v = i.queryMeta(j);
if (!v) { if (!v) {
printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
metaObj.attr(j, nullptr); metaObj[j] = nullptr;
} else { } else {
auto placeholder = metaObj.placeholder(j);
PathSet context; PathSet context;
printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context);
} }
} }
} }
@ -958,6 +962,7 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
throw; throw;
} }
} }
std::cout << topObj.dump(2);
} }

View file

@ -516,7 +516,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
if (!store->isValidPath(info->path) || reregister) { if (!store->isValidPath(info->path) || reregister) {
/* !!! races */ /* !!! races */
if (canonicalise) if (canonicalise)
canonicalisePathMetaData(store->printStorePath(info->path), -1); canonicalisePathMetaData(store->printStorePath(info->path), {});
if (!hashGiven) { if (!hashGiven) {
HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); HashResult hash = hashPath(htSHA256, store->printStorePath(info->path));
info->narHash = hash.first; info->narHash = hash.first;
@ -808,14 +808,20 @@ static void opServe(Strings opFlags, Strings opArgs)
if (GET_PROTOCOL_MINOR(clientVersion) >= 2) if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
settings.maxLogSize = readNum<unsigned long>(in); settings.maxLogSize = readNum<unsigned long>(in);
if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
settings.buildRepeat = readInt(in); auto nrRepeats = readInt(in);
settings.enforceDeterminism = readInt(in); if (nrRepeats != 0) {
throw Error("client requested repeating builds, but this is not currently implemented");
}
// Ignore. It used to be true by default, but also only never had any effect when `nrRepeats == 0`.
// We have already asserted that `nrRepeats` in fact is 0, so we can safely ignore this without
// doing something other than what the client asked for.
auto _enforceDeterminism = readInt(in);
settings.runDiffHook = true; settings.runDiffHook = true;
} }
if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { if (GET_PROTOCOL_MINOR(clientVersion) >= 7) {
settings.keepFailed = (bool) readInt(in); settings.keepFailed = (bool) readInt(in);
} }
settings.printRepeatedBuilds = false;
}; };
while (true) { while (true) {

View file

@ -30,6 +30,10 @@ nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult>
if (b.result) { if (b.result) {
j["startTime"] = b.result->startTime; j["startTime"] = b.result->startTime;
j["stopTime"] = b.result->stopTime; j["stopTime"] = b.result->stopTime;
if (b.result->cpuUser)
j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000;
if (b.result->cpuSystem)
j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000;
} }
res.push_back(j); res.push_back(j);
}, b.path.raw()); }, b.path.raw());

View file

@ -4,10 +4,11 @@
#include "store-api.hh" #include "store-api.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "json.hh"
#include "value-to-json.hh" #include "value-to-json.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
struct CmdEval : MixJSON, InstallableCommand struct CmdEval : MixJSON, InstallableCommand
@ -115,9 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand
} }
else if (json) { else if (json) {
JSONPlaceholder jsonOut(std::cout); std::cout << printValueAsJSON(*state, true, *v, pos, context, false).dump() << std::endl;
printValueAsJSON(*state, true, *v, pos, jsonOut, context, false);
std::cout << std::endl;
} }
else { else {

View file

@ -11,7 +11,6 @@
#include "attr-path.hh" #include "attr-path.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "registry.hh" #include "registry.hh"
#include "json.hh"
#include "eval-cache.hh" #include "eval-cache.hh"
#include "markdown.hh" #include "markdown.hh"
@ -21,6 +20,7 @@
using namespace nix; using namespace nix;
using namespace nix::flake; using namespace nix::flake;
using json = nlohmann::json;
class FlakeCommand : virtual Args, public MixFlakeOptions class FlakeCommand : virtual Args, public MixFlakeOptions
{ {
@ -917,35 +917,44 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
{ {
auto flake = lockFlake(); auto flake = lockFlake();
auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt;
StorePathSet sources; StorePathSet sources;
sources.insert(flake.flake.sourceInfo->storePath); sources.insert(flake.flake.sourceInfo->storePath);
if (jsonRoot)
jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath));
// FIXME: use graph output, handle cycles. // FIXME: use graph output, handle cycles.
std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse; std::function<nlohmann::json(const Node & node)> traverse;
traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj) traverse = [&](const Node & node)
{ {
auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>(); nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr);
for (auto & [inputName, input] : node.inputs) { for (auto & [inputName, input] : node.inputs) {
if (auto inputNode = std::get_if<0>(&input)) { if (auto inputNode = std::get_if<0>(&input)) {
auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional<JSONObject>();
auto storePath = auto storePath =
dryRun dryRun
? (*inputNode)->lockedRef.input.computeStorePath(*store) ? (*inputNode)->lockedRef.input.computeStorePath(*store)
: (*inputNode)->lockedRef.input.fetch(store).first.storePath; : (*inputNode)->lockedRef.input.fetch(store).first.storePath;
if (jsonObj3) if (json) {
jsonObj3->attr("path", store->printStorePath(storePath)); auto& jsonObj3 = jsonObj2[inputName];
sources.insert(std::move(storePath)); jsonObj3["path"] = store->printStorePath(storePath);
traverse(**inputNode, jsonObj3); sources.insert(std::move(storePath));
jsonObj3["inputs"] = traverse(**inputNode);
} else {
sources.insert(std::move(storePath));
traverse(**inputNode);
}
} }
} }
return jsonObj2;
}; };
traverse(*flake.lockFile.root, jsonRoot); if (json) {
nlohmann::json jsonRoot = {
{"path", store->printStorePath(flake.flake.sourceInfo->storePath)},
{"inputs", traverse(*flake.lockFile.root)},
};
std::cout << jsonRoot.dump() << std::endl;
} else {
traverse(*flake.lockFile.root);
}
if (!dryRun && !dstUri.empty()) { if (!dryRun && !dstUri.empty()) {
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);

View file

@ -3,7 +3,7 @@
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "common-args.hh" #include "common-args.hh"
#include "json.hh" #include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
@ -91,10 +91,9 @@ struct MixLs : virtual Args, MixJSON
if (path == "/") path = ""; if (path == "/") path = "";
if (json) { if (json) {
JSONPlaceholder jsonRoot(std::cout);
if (showDirectory) if (showDirectory)
throw UsageError("'--directory' is useless with '--json'"); throw UsageError("'--directory' is useless with '--json'");
listNar(jsonRoot, accessor, path, recursive); std::cout << listNar(accessor, path, recursive);
} else } else
listText(accessor); listText(accessor);
} }

View file

@ -2,10 +2,13 @@
#include "store-api.hh" #include "store-api.hh"
#include "make-content-addressed.hh" #include "make-content-addressed.hh"
#include "common-args.hh" #include "common-args.hh"
#include "json.hh"
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
using nlohmann::json;
struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
{ {
CmdMakeContentAddressed() CmdMakeContentAddressed()
@ -25,6 +28,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
; ;
} }
using StorePathsCommand::run;
void run(ref<Store> srcStore, StorePaths && storePaths) override void run(ref<Store> srcStore, StorePaths && storePaths) override
{ {
auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
@ -33,13 +37,15 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
StorePathSet(storePaths.begin(), storePaths.end())); StorePathSet(storePaths.begin(), storePaths.end()));
if (json) { if (json) {
JSONObject jsonRoot(std::cout); auto jsonRewrites = json::object();
JSONObject jsonRewrites(jsonRoot.object("rewrites"));
for (auto & path : storePaths) { for (auto & path : storePaths) {
auto i = remappings.find(path); auto i = remappings.find(path);
assert(i != remappings.end()); assert(i != remappings.end());
jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second);
} }
auto json = json::object();
json["rewrites"] = jsonRewrites;
std::cout << json.dump();
} else { } else {
for (auto & path : storePaths) { for (auto & path : storePaths) {
auto i = remappings.find(path); auto i = remappings.find(path);

View file

@ -1,12 +1,13 @@
#include "command.hh" #include "command.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "json.hh"
#include "common-args.hh" #include "common-args.hh"
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
struct CmdPathInfo : StorePathsCommand, MixJSON struct CmdPathInfo : StorePathsCommand, MixJSON
@ -86,11 +87,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
pathLen = std::max(pathLen, store->printStorePath(storePath).size()); pathLen = std::max(pathLen, store->printStorePath(storePath).size());
if (json) { if (json) {
JSONPlaceholder jsonRoot(std::cout); std::cout << store->pathInfoToJSON(
store->pathInfoToJSON(jsonRoot,
// FIXME: preserve order? // FIXME: preserve order?
StorePathSet(storePaths.begin(), storePaths.end()), StorePathSet(storePaths.begin(), storePaths.end()),
true, showClosureSize, SRI, AllowInvalid); true, showClosureSize, SRI, AllowInvalid).dump();
} }
else { else {

View file

@ -5,7 +5,6 @@
#include "names.hh" #include "names.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
#include "common-args.hh" #include "common-args.hh"
#include "json.hh"
#include "shared.hh" #include "shared.hh"
#include "eval-cache.hh" #include "eval-cache.hh"
#include "attr-path.hh" #include "attr-path.hh"
@ -13,8 +12,10 @@
#include <regex> #include <regex>
#include <fstream> #include <fstream>
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
using json = nlohmann::json;
std::string wrap(std::string prefix, std::string s) std::string wrap(std::string prefix, std::string s)
{ {
@ -84,7 +85,8 @@ struct CmdSearch : InstallableCommand, MixJSON
auto state = getEvalState(); auto state = getEvalState();
auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; std::optional<nlohmann::json> jsonOut;
if (json) jsonOut = json::object();
uint64_t results = 0; uint64_t results = 0;
@ -151,10 +153,11 @@ struct CmdSearch : InstallableCommand, MixJSON
{ {
results++; results++;
if (json) { if (json) {
auto jsonElem = jsonOut->object(attrPath2); (*jsonOut)[attrPath2] = {
jsonElem.attr("pname", name.name); {"pname", name.name},
jsonElem.attr("version", name.version); {"version", name.version},
jsonElem.attr("description", description); {"description", description},
};
} else { } else {
auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m");
if (results > 1) logger->cout(""); if (results > 1) logger->cout("");
@ -193,6 +196,10 @@ struct CmdSearch : InstallableCommand, MixJSON
for (auto & cursor : installable->getCursors(*state)) for (auto & cursor : installable->getCursors(*state))
visit(*cursor, cursor->getAttrPath(), true); visit(*cursor, cursor->getAttrPath(), true);
if (json) {
std::cout << jsonOut->dump() << std::endl;
}
if (!json && !results) if (!json && !results)
throw Error("no results for the given search term(s)!"); throw Error("no results for the given search term(s)!");
} }

View file

@ -5,10 +5,11 @@
#include "common-args.hh" #include "common-args.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "json.hh"
#include "derivations.hh" #include "derivations.hh"
#include <nlohmann/json.hpp>
using namespace nix; using namespace nix;
using json = nlohmann::json;
struct CmdShowDerivation : InstallablesCommand struct CmdShowDerivation : InstallablesCommand
{ {
@ -48,77 +49,63 @@ struct CmdShowDerivation : InstallablesCommand
drvPaths = std::move(closure); drvPaths = std::move(closure);
} }
{ json jsonRoot = json::object();
JSONObject jsonRoot(std::cout, true);
for (auto & drvPath : drvPaths) { for (auto & drvPath : drvPaths) {
if (!drvPath.isDerivation()) continue; if (!drvPath.isDerivation()) continue;
auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); json& drvObj = jsonRoot[store->printStorePath(drvPath)];
auto drv = store->readDerivation(drvPath); auto drv = store->readDerivation(drvPath);
{ {
auto outputsObj(drvObj.object("outputs")); json& outputsObj = drvObj["outputs"];
outputsObj = json::object();
for (auto & [_outputName, output] : drv.outputs) { for (auto & [_outputName, output] : drv.outputs) {
auto & outputName = _outputName; // work around clang bug auto & outputName = _outputName; // work around clang bug
auto outputObj { outputsObj.object(outputName) }; auto& outputObj = outputsObj[outputName];
outputObj = json::object();
std::visit(overloaded { std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doi) { [&](const DerivationOutput::InputAddressed & doi) {
outputObj.attr("path", store->printStorePath(doi.path)); outputObj["path"] = store->printStorePath(doi.path);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj["hashAlgo"] = dof.hash.printMethodAlgo();
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); outputObj["hash"] = dof.hash.hash.to_string(Base16, false);
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
}, },
[&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) { [&](const DerivationOutput::Impure & doi) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
outputObj.attr("impure", true); outputObj["impure"] = true;
}, },
}, output.raw()); }, output.raw());
} }
} }
{ {
auto inputsList(drvObj.list("inputSrcs")); auto& inputsList = drvObj["inputSrcs"];
inputsList = json::array();
for (auto & input : drv.inputSrcs) for (auto & input : drv.inputSrcs)
inputsList.elem(store->printStorePath(input)); inputsList.emplace_back(store->printStorePath(input));
} }
{ {
auto inputDrvsObj(drvObj.object("inputDrvs")); auto& inputDrvsObj = drvObj["inputDrvs"];
for (auto & input : drv.inputDrvs) { inputDrvsObj = json::object();
auto inputList(inputDrvsObj.list(store->printStorePath(input.first))); for (auto & input : drv.inputDrvs)
for (auto & outputId : input.second) inputDrvsObj[store->printStorePath(input.first)] = input.second;
inputList.elem(outputId);
}
} }
drvObj.attr("system", drv.platform); drvObj["system"] = drv.platform;
drvObj.attr("builder", drv.builder); drvObj["builder"] = drv.builder;
drvObj["args"] = drv.args;
{ drvObj["env"] = drv.env;
auto argsList(drvObj.list("args"));
for (auto & arg : drv.args)
argsList.elem(arg);
}
{
auto envObj(drvObj.object("env"));
for (auto & var : drv.env)
envObj.attr(var.first, var.second);
}
} }
std::cout << jsonRoot.dump(2) << std::endl;
}
std::cout << "\n";
} }
}; };

View file

@ -44,7 +44,7 @@ with import ./config.nix;
}; };
hashmismatch = import <nix/fetchurl.nix> { hashmismatch = import <nix/fetchurl.nix> {
url = "file://" + builtins.getEnv "TMPDIR" + "/dummy"; url = "file://" + builtins.getEnv "TEST_ROOT" + "/dummy";
sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
}; };

View file

@ -40,14 +40,6 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
checkBuildTempDirRemoved $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log
nix build -f check.nix deterministic --rebuild --repeat 1 \
--argstr checkBuildId $checkBuildId --keep-failed --no-link \
2> $TEST_ROOT/log
if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi
# Repeat is set to 1, ie. nix should build deterministic twice.
if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi
checkBuildTempDirRemoved $TEST_ROOT/log
nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
--no-out-link 2> $TEST_ROOT/log --no-out-link 2> $TEST_ROOT/log
checkBuildTempDirRemoved $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log
@ -58,12 +50,6 @@ grep 'may not be deterministic' $TEST_ROOT/log
[ "$status" = "104" ] [ "$status" = "104" ]
checkBuildTempDirRemoved $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log
nix build -f check.nix nondeterministic --rebuild --repeat 1 \
--argstr checkBuildId $checkBuildId --keep-failed --no-link \
2> $TEST_ROOT/log || status=$?
grep 'may not be deterministic' $TEST_ROOT/log
checkBuildTempDirRemoved $TEST_ROOT/log
nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
--no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
grep 'may not be deterministic' $TEST_ROOT/log grep 'may not be deterministic' $TEST_ROOT/log
@ -72,12 +58,6 @@ if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
clearStore clearStore
nix-build dependencies.nix --no-out-link --repeat 3
nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$?
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
path=$(nix-build check.nix -A fetchurl --no-out-link) path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path chmod +w $path
@ -91,13 +71,13 @@ nix-build check.nix -A fetchurl --no-out-link --check
nix-build check.nix -A fetchurl --no-out-link --repair nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]] [[ $(cat $path) != foo ]]
echo 'Hello World' > $TMPDIR/dummy echo 'Hello World' > $TEST_ROOT/dummy
nix-build check.nix -A hashmismatch --no-out-link || status=$? nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ] [ "$status" = "102" ]
echo -n > $TMPDIR/dummy echo -n > $TEST_ROOT/dummy
nix-build check.nix -A hashmismatch --no-out-link nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > $TMPDIR/dummy echo 'Hello World' > $TEST_ROOT/dummy
nix-build check.nix -A hashmismatch --no-out-link --check || status=$? nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ] [ "$status" = "102" ]

68
tests/containers.nix Normal file
View file

@ -0,0 +1,68 @@
# Test whether we can run a NixOS container inside a Nix build using systemd-nspawn.
{ nixpkgs, system, overlay }:
with import (nixpkgs + "/nixos/lib/testing-python.nix") {
inherit system;
extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
};
makeTest ({
name = "containers";
nodes =
{
host =
{ config, lib, pkgs, nodes, ... }:
{ virtualisation.writableStore = true;
virtualisation.diskSize = 2048;
virtualisation.additionalPaths =
[ pkgs.stdenv
(import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel
];
virtualisation.memorySize = 4096;
nix.binaryCaches = lib.mkForce [ ];
nix.extraOptions =
''
extra-experimental-features = nix-command auto-allocate-uids cgroups
extra-system-features = uid-range
'';
nix.nixPath = [ "nixpkgs=${nixpkgs}" ];
};
};
testScript = { nodes }: ''
start_all()
host.succeed("nix --version >&2")
# Test that 'id' gives the expected result in various configurations.
# Existing UIDs, sandbox.
host.succeed("nix build --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1")
host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
# Existing UIDs, no sandbox.
host.succeed("nix build --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2")
host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]")
# Auto-allocated UIDs, sandbox.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3")
host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
# Auto-allocated UIDs, no sandbox.
host.succeed("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4")
host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]")
# Auto-allocated UIDs, UID range, sandbox.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true")
host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]")
# Auto-allocated UIDs, UID range, no sandbox.
host.fail("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true")
# Run systemd-nspawn in a Nix build.
host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}")
host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]")
'';
})

View file

@ -1,7 +1,6 @@
source common.sh source common.sh
enableFeatures "fetch-closure" enableFeatures "fetch-closure"
needLocalStore "'--no-require-sigs' cant be used with the daemon"
clearStore clearStore
clearCacheCache clearCacheCache
@ -28,15 +27,19 @@ clearStore
[ ! -e $nonCaPath ] [ ! -e $nonCaPath ]
[ -e $caPath ] [ -e $caPath ]
# In impure mode, we can use non-CA paths. if [[ "$NIX_REMOTE" != "daemon" ]]; then
[[ $(nix eval --raw --no-require-sigs --impure --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
}
") = $nonCaPath ]]
[ -e $nonCaPath ] # In impure mode, we can use non-CA paths.
[[ $(nix eval --raw --no-require-sigs --impure --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
}
") = $nonCaPath ]]
[ -e $nonCaPath ]
fi
# 'toPath' set to empty string should fail but print the expected path. # 'toPath' set to empty string should fail but print the expected path.
nix eval -v --json --expr " nix eval -v --json --expr "

8
tests/id-test.nix Normal file
View file

@ -0,0 +1,8 @@
{ name, uidRange ? false }:
with import <nixpkgs> {};
runCommand name
{ requiredSystemFeatures = if uidRange then ["uid-range"] else [];
}
"id; id > $out"

78
tests/systemd-nspawn.nix Normal file
View file

@ -0,0 +1,78 @@
{ nixpkgs }:
let
machine = { config, pkgs, ... }:
{
system.stateVersion = "22.05";
boot.isContainer = true;
systemd.services.console-getty.enable = false;
networking.dhcpcd.enable = false;
services.httpd = {
enable = true;
adminAddr = "nixos@example.org";
};
systemd.services.test = {
wantedBy = [ "multi-user.target" ];
after = [ "httpd.service" ];
script = ''
source /.env
echo "Hello World" > $out/msg
ls -lR /dev > $out/dev
${pkgs.curl}/bin/curl -sS --fail http://localhost/ > $out/page.html
'';
unitConfig = {
FailureAction = "exit-force";
FailureActionExitStatus = 42;
SuccessAction = "exit-force";
};
};
};
cfg = (import (nixpkgs + "/nixos/lib/eval-config.nix") {
modules = [ machine ];
system = "x86_64-linux";
});
config = cfg.config;
in
with cfg._module.args.pkgs;
runCommand "test"
{ buildInputs = [ config.system.path ];
requiredSystemFeatures = [ "uid-range" ];
toplevel = config.system.build.toplevel;
}
''
root=$(pwd)/root
mkdir -p $root $root/etc
export > $root/.env
# Make /run a tmpfs to shut up a systemd warning.
mkdir /run
mount -t tmpfs none /run
chmod 0700 /run
mount -t cgroup2 none /sys/fs/cgroup
mkdir -p $out
touch /etc/os-release
echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id
SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 \
${config.systemd.package}/bin/systemd-nspawn \
--keep-unit \
-M ${config.networking.hostName} -D "$root" \
--register=no \
--resolv-conf=off \
--bind-ro=/nix/store \
--bind=$out \
--private-network \
$toplevel/init
''