From d0f2da214badd4cded6fb00e97af4588da49c0d0 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 16 Oct 2022 01:32:55 +0200 Subject: [PATCH 001/113] primops: make nature of foldl' strictness clearer * Clarify the documentation of foldl': That the arguments are forced before application (?) of `op` is necessarily true. What is important to stress is that we force every application of `op`, even when the value turns out to be unused. * Move the example before the comment about strictness to make it less confusing: It is a general example and doesn't really showcase anything about foldl' strictness. * Add test cases which nail down aspects of foldl' strictness: * The initial accumulator value is not forced unconditionally. * Applications of op are forced. * The list elements are not forced unconditionally. --- src/libexpr/primops.cc | 6 +++--- .../lang/eval-fail-foldlStrict-strict-op-application.nix | 5 +++++ tests/lang/eval-okay-foldlStrict-lazy-elements.exp | 1 + tests/lang/eval-okay-foldlStrict-lazy-elements.nix | 9 +++++++++ .../eval-okay-foldlStrict-lazy-initial-accumulator.exp | 1 + .../eval-okay-foldlStrict-lazy-initial-accumulator.nix | 6 ++++++ 6 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-fail-foldlStrict-strict-op-application.nix create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-elements.exp create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-elements.nix create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 28b998474..ba4b7c67a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2908,9 +2908,9 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/lang/eval-fail-foldlStrict-strict-op-application.nix new file mode 100644 index 000000000..1620cc76e --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.nix @@ -0,0 +1,5 @@ +# Tests that the result of applying op is forced even if the value is never used +builtins.foldl' + (_: f: f null) + null + [ (_: throw "Not the final value, but is still forced!") (_: 23) ] diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/lang/eval-okay-foldlStrict-lazy-elements.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-elements.exp @@ -0,0 +1 @@ +42 diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/lang/eval-okay-foldlStrict-lazy-elements.nix new file mode 100644 index 000000000..c666e07f3 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-elements.nix @@ -0,0 +1,9 @@ +# Tests that the rhs argument of op is not forced unconditionally +let + lst = builtins.foldl' + (acc: x: acc ++ [ x ]) + [ ] + [ 42 (throw "this shouldn't be evaluated") ]; +in + +builtins.head lst diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp @@ -0,0 +1 @@ +42 diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix new file mode 100644 index 000000000..abcd5366a --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix @@ -0,0 +1,6 @@ +# Checks that the nul value for the accumulator is not forced unconditionally. +# Some languages provide a foldl' that is strict in this argument, but Nix does not. +builtins.foldl' + (_: x: x) + (throw "This is never forced") + [ "but the results of applying op are" 42 ] From 180538672a017488f9c23a4f72da5e5738a7570c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 5 Dec 2022 11:01:46 +0100 Subject: [PATCH 002/113] disallow selecting shell prompt in code samples this is a quick half-fix for command line examples, as discussed discussed in [1]. [1]: https://github.com/NixOS/nix/pull/7389 examples which look like this $ foo bar baz are confusing for Unix shell beginners, because it's hard to discern what is supposed to be entered into the actual command line when the convention of prefixing `$` is not known, as barely any real-world shell looks that way any more. this change prevents selecting the prompt part with the mouse in the HTML representation of the Nix manual. it does not prevent selecting the output part of the shell example. it also does not address that the copy button provided by mdBook takes the entire sample, including the prompts, into the clipboard. --- doc/manual/custom.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 69d48d4a7..b90f5423f 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -5,3 +5,7 @@ h1:not(:first-of-type) { h2 { margin-top: 1em; } + +.hljs-meta { + user-select: none; +} From aaef82e192fbee108f9c2f64f4dd1fa2cd21d72b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 14:12:05 +0100 Subject: [PATCH 003/113] Document the release process --- maintainers/release-process.md | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 maintainers/release-process.md diff --git a/maintainers/release-process.md b/maintainers/release-process.md new file mode 100644 index 000000000..68063f50c --- /dev/null +++ b/maintainers/release-process.md @@ -0,0 +1,166 @@ +# Nix release process + +## Release artifacts + +The release process is intended to create the following for each +release: + +* A Git tag + +* Binary tarballs in https://releases.nixos.org/?prefix=nix/ + +* Docker images + +* Closures in https://cache.nixos.org + +* (Optionally) Updated `fallback-paths.nix` in Nixpkgs + +* Update the manual on https://nixos.org/manual/nix/stable/ + +## Creating a new release from master + +* Make sure that https://hydra.nixos.org/jobset/nix/master is green. + +* In a checkout of the Nix repo, make sure you're on `master` and do a + `git pull`. + +* Move the contents of `doc/manual/src/release-notes/rl-next.md` + (except the first line) to + `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is + the contents of `.version` *without* the patch level, e.g. `2.12` + rather than `2.12.0`). + +* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like + + ``` + # Release 2.12 (2022-12-06) + ``` + +* Proof-read / edit / rearrange the release notes. Breaking changes + and highlights should go to the top. + +* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` + (*not* `SUMMARY.md`), e.g. + + ``` + - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) + ``` + +* Run + + ```console + $ git checkout -b release-notes + $ git add doc/manual/src/release-notes/rl-$VERSION.md + $ git commit -a -m 'Release notes' + $ git push --set-upstream edolstra release-notes + ``` + +* Create a PR for `release-notes` and auto-merge it. + +* Wait for the PR to be merged. + +* Branch the release: + + ```console + $ git checkout master + $ git pull + $ git checkout -b $VERSION-maintenance + ``` + +* Mark the release as stable: + + ```console + $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + ``` + + This removes the link to `rl-next.md` from the manual and sets + `officialRelease = true` in `flake.nix`. + +* Push the release branch: + + ```console + $ git push + ``` + +* Create a jobset for the release branch on Hydra as follows: + + * Go to the jobset of the previous release + (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). + + * Select `Actions -> Clone this jobset`. + + * Set identifier to `maintenance-$VERSION`. + + * Set description to `$VERSION release branch`. + + * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. + + * Hit `Create jobset`. + +* Wait for the new jobset to evaluate and build. If impatient, go to + the evaluation and select `Actions -> Bump builds to front of + queue`. + +* When the jobset evaluation has succeeded building, take note of the + evaluation ID (e.g. `1780832` in + https://hydra.nixos.org/eval/1780832). + +* Tag the release and upload the release artifacts to + `releases.nixos.org` and dockerhub: + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Note: `IS_LATEST=1` causes the `latest-release` branch to be + force-updated. This is used by the `nixos.org` website to get the + latest Nix manual. + + TODO: This script requires the right AWS credentials. Document. + + TODO: This script currently requires a + `/home/eelco/Dev/nix-pristine` and + `/home/eelco/Dev/nixpkgs-pristine`. + +* Prepare for the next point release by editing `.version` to + e.g. + + ```console + $ echo -n 2.12.1 > .version + $ git commit -a -m 'Bump version' + $ git push + ``` + + Note the `-n`: `.version` must not end in a newline. + + Commit and push this to the maintenance branch. + +* Bump the version of `master`: + + ```console + $ git checkout master + $ echo -n 2.13.0 > .version + $ git checkout -b bump-2.13 + $ git commit -a -m 'Bump version' + $ git push + ``` + + Make a PR and auto-merge it. + +## Creating a point release + +* Wait for the desired evaluation of the maintenance jobset to finish + building. + +* Run + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Omit `IS_LATEST=1` when creating a point release that is not on the + most recent stable branch. This prevents `nixos.org` to going back + to an older release. + +* Bump the version number of the release branch as above (e.g. to + `2.12.2`). From 98364a0df234f0febfddbcdde4a627405d6b19f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 14:33:12 +0100 Subject: [PATCH 004/113] Mention milestone --- maintainers/release-process.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 68063f50c..5eeb0c206 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -147,6 +147,10 @@ release: Make a PR and auto-merge it. +* Create a milestone for the next release, move all unresolved issues + from the previous milestone, and close the previous milestone. Set + the date for the next milestone 6 weeks from now. + ## Creating a point release * Wait for the desired evaluation of the maintenance jobset to finish From f2b9e1f8cb682d37533feed7d5b1d1a4ee13ad81 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 15:26:26 +0100 Subject: [PATCH 005/113] Update maintainers/release-process.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- maintainers/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 5eeb0c206..b266b35e0 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -15,7 +15,7 @@ release: * (Optionally) Updated `fallback-paths.nix` in Nixpkgs -* Update the manual on https://nixos.org/manual/nix/stable/ +* An updated manual on https://nixos.org/manual/nix/stable/ ## Creating a new release from master From 758c2fdbfe93ab5d3582cc83dfef80f20387630b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 15:27:45 +0100 Subject: [PATCH 006/113] Update maintainers/release-process.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- maintainers/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index b266b35e0..e1f8e7d75 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -59,7 +59,7 @@ release: * Wait for the PR to be merged. -* Branch the release: +* Create a branch for the release: ```console $ git checkout master From 31e3c7d6fd5ce6c009ef2b7025aa1de43050ca2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 17:07:51 +0100 Subject: [PATCH 007/113] Mention Discourse --- maintainers/release-process.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index e1f8e7d75..759e80d3c 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -79,7 +79,7 @@ release: * Push the release branch: ```console - $ git push + $ git push --set-upstream origin $VERSION-maintenance ``` * Create a jobset for the release branch on Hydra as follows: @@ -151,6 +151,9 @@ release: from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. +* Post an announcement on Discourse, including the contents of + `rl-$VERSION.md`. + ## Creating a point release * Wait for the desired evaluation of the maintenance jobset to finish From 27392a3b48bad24385ddf1c359f7c04164b6aabc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 17:46:42 +0100 Subject: [PATCH 008/113] More tweaks --- maintainers/release-process.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 759e80d3c..27452b37e 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -139,10 +139,12 @@ release: ```console $ git checkout master - $ echo -n 2.13.0 > .version - $ git checkout -b bump-2.13 + $ git pull + $ NEW_VERSION=2.13.0 + $ echo -n $NEW_VERSION > .version + $ git checkout -b bump-$NEW_VERSION $ git commit -a -m 'Bump version' - $ git push + $ git push --set-upstream origin bump-$NEW_VERSION ``` Make a PR and auto-merge it. From acb69a7beb434e5a5a25462f6d4e38d771eca7a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 13:29:50 +0100 Subject: [PATCH 009/113] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> Co-authored-by: Robert Hensing --- maintainers/release-process.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 27452b37e..565539bf6 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -52,10 +52,10 @@ release: $ git checkout -b release-notes $ git add doc/manual/src/release-notes/rl-$VERSION.md $ git commit -a -m 'Release notes' - $ git push --set-upstream edolstra release-notes + $ git push --set-upstream $REMOTE release-notes ``` -* Create a PR for `release-notes` and auto-merge it. +* Create a PR for `release-notes`. * Wait for the PR to be merged. @@ -122,17 +122,16 @@ release: `/home/eelco/Dev/nix-pristine` and `/home/eelco/Dev/nixpkgs-pristine`. + TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ * Prepare for the next point release by editing `.version` to e.g. ```console - $ echo -n 2.12.1 > .version + $ echo 2.12.1 > .version $ git commit -a -m 'Bump version' $ git push ``` - Note the `-n`: `.version` must not end in a newline. - Commit and push this to the maintenance branch. * Bump the version of `master`: @@ -173,3 +172,8 @@ release: * Bump the version number of the release branch as above (e.g. to `2.12.2`). + +## Recovering from mistakes + +`upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release. + From 6eed1325ffbe50214620e974685e36731e8730b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Dec 2022 17:04:06 +0100 Subject: [PATCH 010/113] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- maintainers/release-process.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 565539bf6..75c1470b8 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -17,11 +17,11 @@ release: * An updated manual on https://nixos.org/manual/nix/stable/ -## Creating a new release from master +## Creating a new release from the `master` branch -* Make sure that https://hydra.nixos.org/jobset/nix/master is green. +* Make sure that the [Hydra `master` jobset](https://hydra.nixos.org/jobset/nix/master) succeeds. -* In a checkout of the Nix repo, make sure you're on `master` and do a +* In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. * Move the contents of `doc/manual/src/release-notes/rl-next.md` @@ -103,10 +103,10 @@ release: * When the jobset evaluation has succeeded building, take note of the evaluation ID (e.g. `1780832` in - https://hydra.nixos.org/eval/1780832). + `https://hydra.nixos.org/eval/1780832`). * Tag the release and upload the release artifacts to - `releases.nixos.org` and dockerhub: + [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/): ```console $ IS_LATEST=1 ./maintainers/upload-release.pl @@ -114,7 +114,7 @@ release: Note: `IS_LATEST=1` causes the `latest-release` branch to be force-updated. This is used by the `nixos.org` website to get the - latest Nix manual. + [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/). TODO: This script requires the right AWS credentials. Document. @@ -146,13 +146,13 @@ release: $ git push --set-upstream origin bump-$NEW_VERSION ``` - Make a PR and auto-merge it. + Make a pull request and auto-merge it. * Create a milestone for the next release, move all unresolved issues from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. -* Post an announcement on Discourse, including the contents of +* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of `rl-$VERSION.md`. ## Creating a point release From 5c0d3b476f617723b90d6e9daf0d8b53ac25bbac Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Jan 2023 00:19:18 +0100 Subject: [PATCH 011/113] release-process: Add "create a backport label" --- maintainers/release-process.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 75c1470b8..b52d218f5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -152,6 +152,8 @@ release: from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. +* Create a backport label + * Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of `rl-$VERSION.md`. From 763c1dfc2b0875251385d9b228c0fc1bfd654bf4 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 16 Jan 2023 11:35:12 -0800 Subject: [PATCH 012/113] Expand installation.md Changes the `quick-start.md` to recommend a multi-user install, since single-user is not supported on MacOS and https://nixos.org/download.html recommends multi-user. Expands `installation.md` to reflect wording on https://nixos.org/download.html --- doc/manual/src/installation/installation.md | 40 +++++++++++++++++++-- doc/manual/src/quick-start.md | 12 +++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/installation.md index b40c5b95f..dafdeb667 100644 --- a/doc/manual/src/installation/installation.md +++ b/doc/manual/src/installation/installation.md @@ -1,2 +1,38 @@ -This section describes how to install and configure Nix for first-time -use. +# Installation + +This section describes how to install and configure Nix for first-time use. + +The current recommended option on Linux and MacOS is [multi-user](#multi-user). + +## Multi-user + +This installation offers better sharing, improved isolation, and more security +over a single user installation. + +This option requires either: + +* Linux running systemd, with SELinux disabled +* MacOS + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +## Single-user + +> Single-user is not supported on Mac. + +This installation has less requirements than the multi-user install, however it +cannot offer equivalent sharing, isolation, or security. + +This option is suitable for systems without systemd. + +```console +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +## Distributions + +The Nix community maintains installers for several distributions. + +They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index b54e73500..e11549984 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -4,16 +4,16 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install single-user Nix by running the following: +1. Install Nix by running the following: ```console - $ bash <(curl -L https://nixos.org/nix/install) + $ bash <(curl -L https://nixos.org/nix/install) --daemon ``` - This will install Nix in `/nix`. The install script will create - `/nix` using `sudo`, so make sure you have sufficient rights. (For - other installation methods, see - [here](installation/installation.md).) + The install script will use `sudo`, so make sure you have sufficient rights. + On Linux, `--daemon` can be omitted for a single-user install. + + For other installation methods, see [here](installation/installation.md). 1. See what installable packages are currently available in the channel: From 3272a2d26fd24647509cddfc2267310416a41cd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Jan 2023 12:16:27 +0100 Subject: [PATCH 013/113] maintainers: Document backporting quirk Closes #7623 --- maintainers/backporting.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 maintainers/backporting.md diff --git a/maintainers/backporting.md b/maintainers/backporting.md new file mode 100644 index 000000000..fe860bc61 --- /dev/null +++ b/maintainers/backporting.md @@ -0,0 +1,8 @@ + +# Backporting + +Backports are handled by the backport action. + +Since GitHub Actions can not trigger actions, the backport PR needs to be re-triggered by another actor. This is achieved by closing and reopening the backport PR. + +This specifically affects the `installer_test` check, but note that it only runs after the other tests, so it may take a while to appear. From 32ca59649b0af8d8d77905274fe63de3a3fc0d1e Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Wed, 1 Feb 2023 20:43:26 +0000 Subject: [PATCH 014/113] nix-shell: Colour the prompt red if the user is root This matches the nixos prompt colours - green for standard user, red for root --- src/nix-build/nix-build.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 049838bb1..da76c2ace 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " From 92edc3836904c2448387be11330a6e7c62593873 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 1 Feb 2023 20:05:56 -0500 Subject: [PATCH 015/113] Don't send plugin-files to the daemon. This is radically unsafe and the daemon has already loaded its plugins anyway. Fixes cachix/devenv#276 --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ff57a77ca..d1296627a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 895c525d04901022b7e0c4586ee12774fb4f8c6f Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 2 Feb 2023 06:03:45 -0500 Subject: [PATCH 016/113] daemon: Warn on old clients passing unexpected plugin-files. The setting itself was already ignored due to exception trying to set pluginFiles. --- src/libstore/daemon.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e2a7dab35..d6621ec0b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -236,6 +236,10 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name From 51013da9211b4f18ca97b7194788121c92094bb1 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 28 Jan 2023 11:48:23 +0100 Subject: [PATCH 017/113] perl: run `initLibStore()` on `openStore()` Since #7478 it's mandatory that `initLibStore()` is called for store operations. However that's not the case when running `openStore()` in Perl using the perl-bindings. That breaks e.g. `hydra-eval-jobset` when built against Nix 2.13 which uses small portions of the store API. --- perl/lib/Nix/Store.xs | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 54ad1680c..de91dc28d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -26,6 +26,7 @@ static ref store() static std::shared_ptr _store; if (!_store) { try { + initLibStore(); loadConfFile(); settings.lockCPU = false; _store = openStore(); From 479c0117840a5dc710019db006c5940b29d98dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 12:02:03 -0500 Subject: [PATCH 018/113] Get rid of the `authHook` parameter on `processConnection` This is (morally) dead code. As @edolstra pointed out in https://github.com/NixOS/nix/pull/5226#discussion_r1073470813, this is no longer needed. I created this in 8d4162ff9e940ea9e2f97b07f3030a722695901a, so it is fitting that I now destroy it :). --- src/libstore/build/local-derivation-goal.cc | 3 +-- src/libstore/daemon.cc | 7 +------ src/libstore/daemon.hh | 7 +------ src/nix/daemon.cc | 11 ++--------- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 572f71045..8ff83f748 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1516,8 +1516,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) {}); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..d1f69fe2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -985,8 +985,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1029,10 +1028,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function authHook); + RecursiveFlag recursive); } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..96568fb8f 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -241,14 +241,7 @@ static void daemonLoop() // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -301,7 +294,7 @@ static void runDaemon(bool stdio) /* Auth hook is empty because in this mode we blindly trust the standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); } } else daemonLoop(); From a47e055e090a85884413dfc7889897f4e3d831fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 13:20:44 -0500 Subject: [PATCH 019/113] Move `trustedUsers` and `allowedUsers` to separate config struct These settings are not needed for libstore at all, they are just used by the nix daemon *command* for authorization on unix domain sockets. My moving them to a new configuration struct just in that file, we avoid them leaking anywhere else. Also, it is good to break up the mammoth `Settings` struct in general. Issue #5638 tracks this. The message is not changed because I do not want to regress in convenience to the user. Just saying "this connection is not trusted" doesn't tell them out to fix the issue. The ideal thing to do would be to somehow parameterize `processCommand` on how the error should be displayed, so different sorts of connections can display different information to the user based on how authentication is performed for the connection in question. This, however, is a good bit more work, so it is left for the future. This came up with me thinking about the tcp:// store (#5265). The larger project is not TCP *per se*, but the idea that it should be possible for something else to manage access control to services like the Nix Daemon, and those services simply trust or trust the incoming connection as they are told. This is a more capability-oriented way of thinking about trust than "every server implements its own auth separately" as we are used to today. Its very great that libstore itself already implements just this model, and so via this refactor I basically want to "enshrine" that so it continues to be the case. --- src/libstore/daemon.cc | 13 +++++++++++-- src/libstore/globals.hh | 34 ++-------------------------------- src/nix/daemon.cc | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..591693a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -529,7 +529,14 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -546,7 +553,9 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c3ccb5e11..42981219d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -279,8 +279,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -696,24 +696,6 @@ public: )", {"trusted-binary-caches"}}; - Setting trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -736,18 +718,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..b391ebed5 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -34,6 +34,43 @@ using namespace nix; using namespace nix::daemon; +struct UserSettings : Config { + + Setting trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +UserSettings userSettings; + +static GlobalConfig::Register rSettings(&userSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -203,8 +240,8 @@ static void daemonLoop() struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; + Strings trustedUsers = userSettings.trustedUsers; + Strings allowedUsers = userSettings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = Trusted; From f20d3726dd9031a4ec311c77f30caa7cc493138b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 29 Jan 2023 15:57:15 +0100 Subject: [PATCH 020/113] advertise transport encoding in http transfers to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tl;dr: With this 1 line change I was able to get a speedup of 1.5x on 1Gbit/s wan connections by enabling zstd compression in nginx. Also nix already supported all common compression format for http transfer, webservers usually only enable them if they are advertised through the Accept-Encoding header. This pull requests makes nix advertises content compression support for zstd, br, gzip and deflate. It's particular useful to add transparent compression for binary caches that serve packages from the host nix store in particular nix-serve, nix-serve-ng and harmonia. I tried so far gzip, brotli and zstd, whereas only zstd was able to bring me performance improvements for 1Gbit/s WAN connections. The following nginx configuration was used in combination with the [zstd module](https://github.com/tokers/zstd-nginx-module) and [harmonia](https://github.com/nix-community/harmonia/) ```nix { services.nginx.virtualHosts."cache.yourhost.com" = { locations."/".extraConfig = '' proxy_pass http://127.0.0.1:5000; proxy_set_header Host $host; proxy_redirect http:// https://; proxy_http_version 1.1; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; zstd on; zstd_types application/x-nix-archive; ''; }; } ``` For testing I unpacked a linux kernel tarball to the nix store using this command `nix-prefetch-url --unpack https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.1.8.tar.gz`. Before: ```console $ nix build && rm -rf /tmp/hello && time ./result/bin/nix copy --no-check-sigs --from https://cache.thalheim.io --to 'file:///tmp/hello?compression=none' '/nix/store/j42mahch5f0jvfmayhzwbb88sw36fvah-linux-6.1.8.tar.gz' warning: Git tree '/scratch/joerg/nix' is dirty real 0m18,375s user 0m2,889s sys 0m1,558s ``` After: ```console $ nix build && rm -rf /tmp/hello && time ./result/bin/nix copy --no-check-sigs --from https://cache.thalheim.io --to 'file:///tmp/hello?compression=none' '/nix/store/j42mahch5f0jvfmayhzwb b88sw36fvah-linux-6.1.8.tar.gz' real 0m11,884s user 0m4,130s sys 0m1,439s ``` Signed-off-by: Jörg Thalheim Update src/libstore/filetransfer.cc Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libstore/filetransfer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 756bd4423..b25089ec3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -101,6 +101,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) From 45fa297e4052acef962d9d124241e7abd02f58af Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Feb 2023 11:21:47 -0500 Subject: [PATCH 021/113] Factor out `InstallableStorePath` to its own file, dedup `nix app` had something called `InstallableDerivedPath` which is actually the same thing. We go with the later's name because it has become more correct. I originally did this change (more hurriedly) as part of #6225 --- a mini store-only Nix and a full Nix need to share this code. In the first RFC meeting for https://github.com/NixOS/rfcs/pull/134 we discussed how some splitting of the massive `installables.cc` could begin prior, as that is a good thing anyways. (@edolstra's words, not mine!) This would be one such PR. --- src/libcmd/installable-derived-path.cc | 70 ++++++++++++++++++++++++++ src/libcmd/installable-derived-path.hh | 28 +++++++++++ src/libcmd/installables.cc | 70 ++------------------------ src/nix/app.cc | 27 +--------- 4 files changed, 104 insertions(+), 91 deletions(-) create mode 100644 src/libcmd/installable-derived-path.cc create mode 100644 src/libcmd/installable-derived-path.hh diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc new file mode 100644 index 000000000..a9921b901 --- /dev/null +++ b/src/libcmd/installable-derived-path.cc @@ -0,0 +1,70 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{.path = derivedPath, .info = {} }}; +} + +std::optional InstallableDerivedPath::getStorePath() +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + [&](const DerivedPath::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more liberally. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + // First, we accept a symlink chain or an actual store path. + auto storePath = store->followLinksToStorePath(prefix); + // Second, we see if the store path ends in `.drv` to decide what sort + // of derived path they want. + // + // This handling predates the `^` syntax. The `^*` in + // `/nix/store/hash-foo.drv^*` unambiguously means "do the + // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could + // also unambiguously mean "do the DerivedPath::Opaque` case". + // + // Issue #7261 tracks reconsidering this `.drv` dispatching. + return storePath.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = std::move(storePath), + .outputs = OutputsSpec::All {}, + } + : (DerivedPath) DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh new file mode 100644 index 000000000..042878b91 --- /dev/null +++ b/src/libcmd/installable-derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional getStorePath() override; + + static InstallableDerivedPath parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 24f458f1a..5a75ed7af 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,6 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" #include "outputs-spec.hh" #include "util.hh" #include "command.hh" @@ -396,38 +397,6 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref store; - DerivedPath req; - - InstallableStorePath(ref store, DerivedPath && req) - : store(store), req(std::move(req)) - { } - - std::string what() const override - { - return req.to_string(*store); - } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{.path = req, .info = {} }}; - } - - std::optional getStorePath() override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, req.raw()); - } -}; - struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; @@ -792,41 +761,10 @@ std::vector> SourceExprCommand::parseInstallables( auto prefix = std::move(prefix_); auto extendedOutputsSpec = std::move(extendedOutputsSpec_); - auto found = prefix.find('/'); - if (found != std::string::npos) { + if (prefix.find('/') != std::string::npos) { try { - auto derivedPath = std::visit(overloaded { - // If the user did not use ^, we treat the output more liberally. - [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { - // First, we accept a symlink chain or an actual store path. - auto storePath = store->followLinksToStorePath(prefix); - // Second, we see if the store path ends in `.drv` to decide what sort - // of derived path they want. - // - // This handling predates the `^` syntax. The `^*` in - // `/nix/store/hash-foo.drv^*` unambiguously means "do the - // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could - // also unambiguously mean "do the DerivedPath::Opaque` case". - // - // Issue #7261 tracks reconsidering this `.drv` dispatching. - return storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = OutputsSpec::All {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), - }; - }, - // If the user did use ^, we just do exactly what is written. - [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { - return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), - .outputs = outputSpec, - }; - }, - }, extendedOutputsSpec.raw()); - result.push_back(std::make_shared(store, std::move(derivedPath))); + result.push_back(std::make_shared( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { } catch (...) { diff --git a/src/nix/app.cc b/src/nix/app.cc index 08cd0ccd4..5cd65136f 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,4 +1,5 @@ #include "installables.hh" +#include "installable-derived-path.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -8,30 +9,6 @@ namespace nix { -struct InstallableDerivedPath : Installable -{ - ref store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{derivedPath}}; - } - - std::optional getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -146,7 +123,7 @@ App UnresolvedApp::resolve(ref evalStore, ref store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared(store, ctxElt)); + std::make_shared(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); From 6352e20bc8dba9477e25eedbe7bea91cbe1614f9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Feb 2023 18:26:43 -0500 Subject: [PATCH 022/113] Remove `--derivation` from test It doesn't do anything here, and in the next commit `show-derivation will no longer accept this flag. --- tests/ca/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 92f8b429a..cc225c6c8 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1) -nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 +nix --experimental-features 'nix-command ca-derivations' show-derivation "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 From 44bea52ae3ca9569250eb1f50100f2c3260cc688 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Feb 2023 18:27:17 -0500 Subject: [PATCH 023/113] Scope down `--derivation` to just the commands that use it Per the old FIXME, this flag was on too many commands, and mostly ignored. Now it is just on the commands where it actually has an effect. Per https://github.com/NixOS/nix/issues/7261, I would still like to get rid of it entirely, but that is a separate project. This change should be good with or without doing that. --- src/libcmd/command.cc | 10 ++++++++++ src/libcmd/command.hh | 12 ++++++++---- src/libcmd/installables.cc | 7 ------- src/nix/diff-closures.cc | 2 +- src/nix/why-depends.cc | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0740ea960..517cdf617 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -127,6 +127,16 @@ ref EvalCommand::getEvalState() return ref(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 3b4b40981..d16bdbc4b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -96,9 +96,6 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions std::optional expr; bool readOnlyMode = false; - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; - SourceExprCommand(bool supportReadOnlyMode = false); std::vector> parseInstallables( @@ -153,8 +150,15 @@ private: std::string _installable{"."}; }; +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + /* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 24f458f1a..ff8261b09 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -167,13 +167,6 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) .handler = {&expr} }); - addFlag({ - .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, - }); - if (supportReadOnlyMode) { addFlag({ .longName = "read-only", diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0621d662c..3489cc132 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -106,7 +106,7 @@ void printClosureDiff( using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..a3a9dc698 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -27,7 +27,7 @@ static std::string filterPrintable(const std::string & s) return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; From b8e96351f761ebc688f2f2f6a13345576c008396 Mon Sep 17 00:00:00 2001 From: Peter Becich Date: Sun, 5 Feb 2023 12:00:48 -0800 Subject: [PATCH 024/113] fix `clang11Stdenv` dev shell documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clang11StdenvPackages` does not exist ``` │ └───x86_64-linux │ ├───ccacheStdenv: development environment 'nix' │ ├───clang11Stdenv: development environment 'nix' │ ├───clangStdenv: development environment 'nix' │ ├───default: development environment 'nix' │ ├───gccStdenv: development environment 'nix' │ ├───libcxxStdenv: development environment 'nix' │ └───stdenv: development environment 'nix' ``` --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9dbafcc0a..e2046acc8 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -45,13 +45,13 @@ To get a shell with a different compilation environment (e.g. stdenv, gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): ```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +$ nix-shell -A devShells.x86_64-linux.clang11Stdenv ``` or if you have a flake-enabled nix: ```console -$ nix develop .#clang11StdenvPackages +$ nix develop .#clang11Stdenv ``` Note: you can use `ccacheStdenv` to drastically improve rebuild From 3050005211951a1053e79634b1eb2fee1ad30a90 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 6 Feb 2023 17:53:03 +0100 Subject: [PATCH 025/113] build-remote: don't warn when all local build slots are taken Previously, build-remote would show a warning if all build slots were taken, even if they would open up later. This caused a lot of spam in the logs. Disable this warning when maxJobs > 0. See #6263 --- src/build-remote/build-remote.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..e197af847 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -112,10 +112,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = settings.maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -214,7 +218,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep(", ", m.supportedFeatures) % concatStringsSep(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } From 6fdce7a9df0af7ea6266ded6c287b1dfc9d3eada Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 22:00:49 +0000 Subject: [PATCH 026/113] Bump cachix/install-nix-action from 18 to 19 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 18 to 19. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v18...v19) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dafba6d85..325579a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' @@ -58,7 +58,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 From 9670cac1ab0119628916dfd5a7df5751ee27530a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 7 Feb 2023 11:11:50 +0100 Subject: [PATCH 027/113] Backport PR title: Bot-based -> Automatic In hope to reduce friction in contributor-facing communication, use a more common word. "bot" needless jargon for "something the machine does itself". --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ca5af260f..558cfa804 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -27,6 +27,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 From 2915db7b28ffab0aee9cfc3759f1ad0ee11b4cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 7 Feb 2023 11:38:09 +0100 Subject: [PATCH 028/113] doc: fixup 7714 --- .../src/language/advanced-attributes.md | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index a164b3997..5a63236e5 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -260,7 +260,7 @@ Derivations can declare some infrequently used optional attributes. If the special attribute `__structuredAttrs` is set to `true`, the other derivation attributes are serialised in JSON format and made available to the builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for `passAsFile` since JSON files + directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see @@ -282,11 +282,13 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the output path. + - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. + Example: + ```nix __structuredAttrs = true; @@ -305,9 +307,20 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - When using [structured attributes](#adv-attr-structuredAttrs), the **experimental** - attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`, - disables scanning the build output for runtime dependencies altogether. + > **Warning** + > This is an experimental feature. + > + > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = discard-references + > ``` + + When using [structured attributes](#adv-attr-structuredAttrs), the + attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. + If set to `true`, it disables scanning the output for runtime dependencies. + + Example: ```nix __structuredAttrs = true; @@ -317,5 +330,3 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - - This is only allowed if the `discard-references` experimental feature is enabled. From 631ba6442a2fd2475b213cd463ca811d21761f36 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 7 Feb 2023 12:08:00 +0100 Subject: [PATCH 029/113] build-remote: store maxBuildJobs before forcing it to 1 --- src/build-remote/build-remote.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e197af847..174435e7c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -114,7 +115,7 @@ static int main_build_remote(int argc, char * * argv) /* It would be possible to build locally after some builds clear out, so don't show the warning now: */ - bool couldBuildLocally = settings.maxBuildJobs > 0 + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); From c20394245d602d43c7a601253bebdf0fab498b52 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Jan 2023 15:48:38 +0100 Subject: [PATCH 030/113] add "instantiate" to glossary --- doc/manual/src/glossary.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 6004df833..d0aff34e2 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -19,6 +19,13 @@ [store derivation]: #gloss-store-derivation + - [instantiate]{#gloss-instantiate}, instantiation\ + Translate a [derivation] into a [store derivation]. + + See [`nix-instantiate`](./command-ref/nix-instantiate.md). + + [instantiate]: #gloss-instantiate + - [realise]{#gloss-realise}, realisation\ Ensure a [store path] is [valid][validity]. From 81e75e4bf6083046155a0c1fd6e312b43a60f7da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Feb 2023 21:42:42 +0100 Subject: [PATCH 031/113] Add some progress indication when fetching submodules --- src/libfetchers/git.cc | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f7d7c07d..fb80d248c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -615,15 +615,24 @@ struct GitInputScheme : InputScheme AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + { + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", tmpDir)); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + } filter = isNotDotGitDirectory; } else { From 2edd5cf6184a7955ba82a2aed8bbfa870bfeb15f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 15:35:32 +0100 Subject: [PATCH 032/113] Fix the origin URL used for fetching submodules We cannot use 'actualUrl', because for file:// repos that's not the original URL that the repo was fetched from. This is a problem since submodules may be relative to the original URL. Fixes e.g. nix eval --impure --json --expr 'builtins.fetchTree { type = "git"; url = "/path/to/blender"; submodules = true; }' where /path/to/blender is a clone of https://github.com/blender/blender.git (which has several relative submodules like '../blender-addons.git'). --- src/libfetchers/git.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index fb80d248c..8981476e1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -627,7 +627,17 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); + /* Ensure that we use the correct origin for fetching + submodules. This matters for submodules with relative + URLs. */ + if (isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + + /* Restore the config.bare setting we may have just + nuked. */ + runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); + } else + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); From a8fe0dc16c18c260a1bb9e88f467e4c929cf22dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 15:50:35 +0100 Subject: [PATCH 033/113] Speed up fetching submodules Previously we would completely refetch the submodules from the network, even though the repo might already have them. Now we copy the .git/modules directory from the repo as an optimisation. This speeds up evaluating builtins.fetchTree { type = "git"; url = "/path/to/blender"; submodules = true; } (where /path/to/blender already has the needed submodules) from 121s to 57s. This is still pretty inefficient and a hack, but a better solution is best done on the lazy-trees branch. This change also help in the case where the repo already has the submodules but the origin is unfetchable for whatever reason (e.g. there have been cases where Nix in a GitHub action doesn't have the right authentication set up). --- src/libfetchers/git.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8981476e1..9908db214 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -639,6 +639,14 @@ struct GitInputScheme : InputScheme } else runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); From 72b18f05a22f46f383e0476262dc363ac0318aa6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 7 Feb 2023 16:36:10 +0100 Subject: [PATCH 034/113] Add a basic daemon authorization test --- flake.nix | 2 + tests/nixos/authorization.nix | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 tests/nixos/authorization.nix diff --git a/flake.nix b/flake.nix index 0325f33b1..7f0b738f8 100644 --- a/flake.nix +++ b/flake.nix @@ -499,6 +499,8 @@ }; # System tests. + tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix new file mode 100644 index 000000000..4b13ec65e --- /dev/null +++ b/tests/nixos/authorization.nix @@ -0,0 +1,79 @@ +{ + name = "authorization"; + + nodes.machine = { + virtualisation.writableStore = true; + # TODO add a test without allowed-users setting. allowed-users is uncommon among NixOS users. + nix.settings.allowed-users = ["alice" "bob"]; + nix.settings.trusted-users = ["alice"]; + + users.users.alice.isNormalUser = true; + users.users.bob.isNormalUser = true; + users.users.mallory.isNormalUser = true; + + nix.settings.experimental-features = "nix-command"; + }; + + testScript = + let + pathFour = "/nix/store/20xfy868aiic0r0flgzq4n5dq1yvmxkn-four"; + in + '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + exec 1>&2 + echo kSELDhobKaF8/VdxIxdP7EQe+Q > one + diff $(nix store add-file one) one + """) + machine.succeed(""" + su --login alice -c ' + set -x + cd ~ + echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two + ls + diff $(echo $(nix store add-file two)) two' 1>&2 + """) + machine.succeed(""" + su --login bob -c ' + set -x + cd ~ + echo 0Jw8RNp7cK0W2AdNbcquofcOVk > three + diff $(nix store add-file three) three + ' 1>&2 + """) + + # We're going to check that a path is not created + machine.succeed(""" + ! [[ -e ${pathFour} ]] + """) + machine.succeed(""" + su --login mallory -c ' + set -x + cd ~ + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four; + (! diff $(nix store add-file four) four 2>&1) | grep -F "cannot open connection to remote store" + (! diff $(nix store add-file four) four 2>&1) | grep -F "Connection reset by peer" + ! [[ -e ${pathFour} ]] + ' 1>&2 + """) + + # Check that the file _can_ be added, and matches the expected path we were checking + machine.succeed(""" + exec 1>&2 + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four + four="$(nix store add-file four)" + diff $four four + diff <(echo $four) <(echo ${pathFour}) + """) + + machine.succeed(""" + su --login alice -c 'nix-store --verify --repair' + """) + + machine.succeed(""" + set -x + su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 + grep -F "you are not privileged to repair paths" diag + """) + ''; +} From 7a6daf61e8a3b6fd91e6d89334fe6b59fe07a2a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:22:50 +0100 Subject: [PATCH 035/113] Fix activity message --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9908db214..3871cf0dd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -620,7 +620,7 @@ struct GitInputScheme : InputScheme // TODO: repoDir might lack the ref (it only checks if rev // exists, see FIXME above) so use a big hammer and fetch // everything to ensure we get the rev. - Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", tmpDir)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); } @@ -648,7 +648,7 @@ struct GitInputScheme : InputScheme } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoDir)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); } From 9a7dc5d71879a32ff06305aa843abf299b39c90f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:46:25 +0100 Subject: [PATCH 036/113] Add some tests --- tests/fetchGitSubmodules.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 50da4cb97..08ccaa3cd 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -104,3 +104,28 @@ noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] + +# Test relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub +initGitRepo $rootRepo +git -C $rootRepo submodule add ../gitSubmodulesSub sub +git -C $rootRepo commit -m "Add submodule" +rev2=$(git -C $rootRepo rev-parse HEAD) +pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") +diff -r -x .gitmodules $pathWithSubmodules $pathWithRelative + +# Test clones that have an upstream with relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo +git clone $rootRepo $cloneRepo +pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathIndirect = $pathWithRelative ]] + +# Test that if the clone has the submodule already, we're not fetching +# it again. +git -C $cloneRepo submodule update --init +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $subRepo +pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathSubmoduleGone = $pathWithRelative ]] From 0a82d2ca491507f727d44a96f07d48934862da1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:48:31 +0100 Subject: [PATCH 037/113] Ask release notes for incompatible changes, not bug fixes --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 344f9405f..db69e51db 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,4 +25,4 @@ Maintainers: tick if completed or explain if not relevant - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - - [ ] new feature or bug fix: updated release notes + - [ ] new feature or incompatible change: updated release notes From fb2f7f5dcc6b37a4f39f59d9f477d3fa57d79095 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jan 2023 17:31:27 +0100 Subject: [PATCH 038/113] Fix auto-uid-allocation in Docker containers This didn't work because sandboxing doesn't work in Docker. However, the sandboxing check is done lazily - after clone(CLONE_NEWNS) fails, we retry with sandboxing disabled. But at that point, we've already done UID allocation under the assumption that user namespaces are enabled. So let's get rid of the "goto fallback" logic and just detect early whether user / mount namespaces are enabled. This commit also gets rid of a compatibility hack for some ancient Linux kernels (<2.13). --- src/libstore/build/local-derivation-goal.cc | 82 +++++---------------- src/libutil/namespaces.cc | 63 ++++++++++++++++ src/libutil/namespaces.hh | 9 +++ 3 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 src/libutil/namespaces.cc create mode 100644 src/libutil/namespaces.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ff83f748..16955e326 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -16,6 +16,7 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "namespaces.hh" #include #include @@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support mount namespaces, which are required for sandboxing"); + debug("auto-disabling sandboxing because mount namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -888,12 +901,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -920,64 +928,15 @@ void LocalDerivationGoal::startBuilder() flags |= CLONE_NEWUSER; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + + if (child == -1) + throw SysError("creating sandboxed builder process using clone()"); writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1045,9 +1004,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc new file mode 100644 index 000000000..0c3c3cbdd --- /dev/null +++ b/src/libutil/namespaces.cc @@ -0,0 +1,63 @@ +#include "namespaces.hh" +#include "util.hh" + +#if __linux__ + +namespace nix { + +bool userNamespacesSupported() +{ + static bool res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWUSER); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +bool mountNamespacesSupported() +{ + static bool res = [&]() -> bool + { + bool useUserNamespace = userNamespacesSupported(); + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWNS | (useUserNamespace ? CLONE_NEWUSER : 0)); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +} + +#endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh new file mode 100644 index 000000000..4ed6cb683 --- /dev/null +++ b/src/libutil/namespaces.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +bool userNamespacesSupported(); + +bool mountNamespacesSupported(); + +} From bc1d9fd8b5a14334af1d0455e6b4d595cae959d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 15:25:56 +0100 Subject: [PATCH 039/113] Check whether we can use PID namespaces In unprivileged podman containers, /proc is not fully visible (there are other filesystems mounted on subdirectories of /proc). Therefore we can't mount a new /proc in the sandbox that matches the PID namespace of the sandbox. So this commit automatically disables sandboxing if /proc is not fully visible. --- src/libstore/build/local-derivation-goal.cc | 6 ++-- src/libutil/namespaces.cc | 37 ++++++++++++++++++--- src/libutil/namespaces.hh | 2 ++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 16955e326..a99da0b5f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -209,10 +209,10 @@ void LocalDerivationGoal::tryLocalBuild() #if __linux__ if (useChroot) { - if (!mountNamespacesSupported()) { + if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support mount namespaces, which are required for sandboxing"); - debug("auto-disabling sandboxing because mount namespaces are not available"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } } diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 0c3c3cbdd..222f0d11b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,8 @@ #include "namespaces.hh" #include "util.hh" +#include "finally.hh" + +#include #if __linux__ @@ -7,10 +10,10 @@ namespace nix { bool userNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { if (!pathExists("/proc/self/ns/user")) { - notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); return false; } @@ -18,7 +21,7 @@ bool userNamespacesSupported() if (!pathExists(maxUserNamespaces) || trim(readFile(maxUserNamespaces)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); return false; } @@ -26,7 +29,7 @@ bool userNamespacesSupported() if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); return false; } @@ -43,7 +46,7 @@ bool userNamespacesSupported() bool mountNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { bool useUserNamespace = userNamespacesSupported(); @@ -58,6 +61,30 @@ bool mountNamespacesSupported() return res; } +bool pidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + /* Check whether /proc is fully visible, i.e. there are no + filesystems mounted on top of files inside /proc. If this + is not the case, then we cannot mount a new /proc inside + the sandbox that matches the sandbox's PID namespace. + See https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return false; + Finally delFP = [&]() { fclose(fp); }; + + while (auto ent = getmntent(fp)) + if (hasPrefix(std::string_view(ent->mnt_dir), "/proc/")) { + debug("PID namespaces do not work because /proc is not fully visible; disabling sandboxing"); + return false; + } + + return true; + }(); + return res; +} + } #endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 4ed6cb683..ad7bb559e 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -6,4 +6,6 @@ bool userNamespacesSupported(); bool mountNamespacesSupported(); +bool pidNamespacesSupported(); + } From d834de2894b5addc5a4a8c5088debd56a8517db1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:01 +0100 Subject: [PATCH 040/113] Fix macOS build --- src/libutil/namespaces.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 222f0d11b..b1cdbfe03 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,11 +1,11 @@ +#if __linux__ + #include "namespaces.hh" #include "util.hh" #include "finally.hh" #include -#if __linux__ - namespace nix { bool userNamespacesSupported() From 4e61877b5c64096fa3ea63bf5ead7e17e1ddef66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:31 +0100 Subject: [PATCH 041/113] More #ifdef --- src/libutil/namespaces.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index ad7bb559e..34e54d5ad 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -2,10 +2,14 @@ namespace nix { +#if __linux__ + bool userNamespacesSupported(); bool mountNamespacesSupported(); bool pidNamespacesSupported(); +#endif + } From c5c0617d6fe6810c35ec56d3116ef523f3f38904 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:59:46 +0100 Subject: [PATCH 042/113] Mention --no-sandbox if sandboxing is unsupported --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a99da0b5f..e1cc504f8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -211,7 +211,7 @@ void LocalDerivationGoal::tryLocalBuild() if (useChroot) { if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } From 0a70b411e1afaa22d8b01560de908246042daf10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 23:01:39 +0100 Subject: [PATCH 043/113] Print debug message if a namespace test fails --- src/libutil/namespaces.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index b1cdbfe03..fdd52d92b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -39,7 +39,12 @@ bool userNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("user namespaces do not work on this system"); + + return supported; }(); return res; } @@ -56,7 +61,12 @@ bool mountNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("mount namespaces do not work on this system"); + + return supported; }(); return res; } From 32db5e63494436b67d71d9393220dcafc34e5227 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Thu, 26 Jan 2023 22:50:36 +0100 Subject: [PATCH 044/113] docs: Fix broken anchor link --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 1f918bd4d..208d65839 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -116,7 +116,7 @@ The result is a string. [store path]: ../glossary.md#gloss-store-path [store]: ../glossary.md#gloss-store -[Path and string concatenation]: #path-and-string-concatenation +[String and path concatenation]: #string-and-path-concatenation ## Update @@ -133,7 +133,7 @@ If an attribute name is present in both, the attribute value from the latter is Comparison is -- [arithmetic] for [number]s +- [arithmetic] for [number]s - lexicographic for [string]s and [path]s - item-wise lexicographic for [list]s: elements at the same index in both lists are compared according to their type and skipped if they are equal. From d910dfe97892dcf26e83b6d591d46fbdfe2c0944 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 31 Jan 2023 14:20:51 +0100 Subject: [PATCH 045/113] docs: Fix formatting of || operator This is a workaround for [mdBook#2000](https://github.com/rust-lang/mdBook/issues/2000) --- doc/manual/src/language/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 208d65839..90b325597 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -24,7 +24,7 @@ | [Equality] | *expr* `==` *expr* | none | 11 | | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | -| Logical disjunction (`OR`) | *bool* `\|\|` *bool* | left | 13 | +| Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | none | 14 | [string]: ./values.md#type-string From 9aeaf98c4b0e99a3af45517a2a54f7715a396542 Mon Sep 17 00:00:00 2001 From: David Dunn <26876072+doubledup@users.noreply.github.com> Date: Wed, 8 Feb 2023 00:32:27 +0200 Subject: [PATCH 046/113] Make install command in documentation compatible with fish shell (#7474) Use a pipe for all install commands Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 2 +- doc/manual/src/installation/env-variables.md | 4 ++-- doc/manual/src/installation/installing-binary.md | 8 ++++---- doc/manual/src/quick-start.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index e2046acc8..53f49602b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -219,7 +219,7 @@ After the CI run completes, you can check the output to extract the installer UR 5. To generate an install command, plug this `install_url` and your GitHub username into this template: ```console - sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve ```