Merge remote-tracking branch 'upstream/master' into path-info

.github/CODEOWNERS

@@ -5,7 +5,7 @@
 # For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/
 
 # Default reviewers if nothing else matches
-* @edolstra @thufschmitt
+* @edolstra
 
 # This file
 .github/CODEOWNERS @edolstra
@@ -13,3 +13,6 @@
 # Public documentation
 /doc @fricklerhandwerk
 *.md @fricklerhandwerk
+
+# Libstore layer
+/src/libstore @thufschmitt

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,4 +25,4 @@ Maintainers: tick if completed or explain if not relevant
  - [ ] documentation in the manual
  - [ ] code and comments are self-explanatory
  - [ ] commit message explains why the change was made
- - [ ] new feature or bug fix: updated release notes
+ - [ ] new feature or incompatible change: updated release notes

.github/workflows/backport.yml

@@ -21,12 +21,12 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.1.0
+        uses: zeebe-io/backport-action@v1.2.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}
           github_workspace: ${{ github.workspace }}
           pull_description: |-
-            Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}.
+            Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}.
           # should be kept in sync with `uses`
           version: v0.0.5

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v19
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'
@@ -58,7 +58,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v19
     - uses: cachix/cachix-action@v12
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -77,7 +77,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v19
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -102,7 +102,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v19
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v12

.gitignore

@@ -75,7 +75,7 @@ perl/Makefile.config
 
 # /tests/
 /tests/test-tmp
-/tests/common.sh
+/tests/common/vars-and-functions.sh
 /tests/result*
 /tests/restricted-innocent
 /tests/shell
@@ -103,6 +103,7 @@ outputs/
 
 *.a
 *.o
+*.o.tmp
 *.so
 *.dylib
 *.dll

Makefile

@@ -36,4 +36,4 @@ endif
 
 include mk/lib.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++20 -I src
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src

README.md

@@ -20,8 +20,8 @@ Information on additional installation methods is available on the [Nix download
 
 ## Building And Developing
 
-See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to
+See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to
-build nix from source with nix-build or how to get a development environment.
+to set up a development environment and build Nix from source.
 
 ## Additional Resources
 

doc/manual/custom.css

@@ -5,3 +5,7 @@ h1:not(:first-of-type) {
 h2 {
     margin-top: 1em;
 }
+
+.hljs-meta {
+    user-select: none;
+}

doc/manual/local.mk

@@ -51,13 +51,13 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
-	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	@# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
 	$(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g'
 	@mv $@.tmp $@
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	@# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \
 	  | sed -e 's^@docroot@^..^g'>> $@.tmp
 	@mv $@.tmp $@
@@ -72,7 +72,7 @@ $(d)/conf-file.json: $(bindir)/nix
 
 $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
-	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	@# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \
 	  | sed -e 's^@docroot@^..^g' >> $@.tmp
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp

doc/manual/src/SUMMARY.md.in

@@ -67,6 +67,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)
   - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
   - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md)

doc/manual/src/command-ref/env-common.md

@@ -91,3 +91,16 @@ Most Nix commands interpret the following environment variables:
     variable sets the initial size of the heap in bytes. It defaults to
     384 MiB. Setting it to a low value reduces memory consumption, but
     will increase runtime due to the overhead of garbage collection.
+
+## XDG Base Directory
+
+New Nix commands conform to the [XDG Base Directory Specification], and use the following environment variables to determine locations of various state and configuration files:
+
+- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`)
+- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`)
+- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`)
+
+Classic Nix commands can also be made to follow this standard using the [`use-xdg-base-directories`] configuration option.
+
+[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
+[`use-xdg-base-directories`]: ../command-ref/conf-file.md#conf-use-xdg-base-directories

doc/manual/src/contributing/cli-guideline.md

@@ -389,6 +389,88 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should
 happen when TTY is not detected on STDERR. We should not display progress /
 status section, but only print warnings and errors.
 
+## Returning future proof JSON
+
+The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this.
+
+Two definitions are helpful here, because while JSON only defines one "key-value"
+object type, we use it to cover two use cases:
+
+ - **dictionary**: a map from names to value that all have the same type. In
+   C++ this would be a `std::map` with string keys.
+ - **record**: a fixed set of attributes each with their own type. In C++, this
+   would be represented by a `struct`.
+
+It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type.
+
+This leads to the following guidelines:
+
+ - The top-level (root) value must be a record.
+
+   Otherwise, one can not change the structure of a command's output.
+
+ - The value of a dictionary item must be a record.
+
+   Otherwise, the item type can not be extended.
+
+ - List items should be records.
+
+   Otherwise, one can not change the structure of the list items.
+
+   If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records.
+
+ - Streaming JSON should return records.
+
+   An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records.
+
+### Examples
+
+
+This is bad, because all keys must be assumed to be store implementations:
+
+```json
+{
+  "local": { ... },
+  "remote": { ... },
+  "http": { ... }
+}
+```
+
+This is good, because the it is extensible at the root, and is somewhat self-documenting:
+
+```json
+{
+  "storeTypes": { "local": { ... }, ... },
+  "pluginSupport": true
+}
+```
+
+While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information.
+For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing.
+
+
+
+The following representation is bad because it is not extensible:
+
+```json
+{ "outputs": [ "out" "bin" ] }
+```
+
+However, simply converting everything to records is not enough, because the order of outputs must be preserved:
+
+```json
+{ "outputs": { "bin": {}, "out": {} } }
+```
+
+The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work.
+While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries.
+
+This representation is extensible and preserves the ordering:
+
+```json
+{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] }
+```
+
 ## Dialog with the user
 
 CLIs don't always make it clear when an action has taken place. For every

doc/manual/src/contributing/hacking.md

@@ -8,25 +8,64 @@ $ git clone https://github.com/NixOS/nix.git
 $ cd nix
 ```
 
-To build Nix for the current operating system/architecture use
+The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions].
+
+[installation instructions]: ../installation/installation.md
+
+## Nix with flakes
+
+This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes.
+
+[flakes]: ../command-ref/new-cli/nix3-flake.md#description
+
+To build all dependencies and start a shell in which all environment
+variables are set up so that those dependencies can be found:
 
 ```console
-$ nix-build
+$ nix develop
 ```
 
-or if you have a flake-enabled nix:
+This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it.
+
+To get a shell with one of the other [supported compilation environments](#compilation-environments):
+
+```console
+$ nix develop .#native-clang11StdenvPackages
+```
+
+> **Note**
+>
+> Use `ccacheStdenv` to drastically improve rebuild time.
+> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`.
+
+To build Nix itself in this shell:
+
+```console
+[nix-shell]$ ./bootstrap.sh
+[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out
+[nix-shell]$ make -j $NIX_BUILD_CORES
+```
+
+To install it in `$(pwd)/outputs` and test it:
+
+```console
+[nix-shell]$ make install
+[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
+[nix-shell]$ nix --version
+nix (Nix) 2.12
+```
+
+To build a release version of Nix:
 
 ```console
 $ nix build
 ```
 
-This will build `defaultPackage` attribute defined in the `flake.nix`
+You can also build Nix for one of the [supported target platforms](#target-platforms).
-file. To build for other platforms add one of the following suffixes to
-it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e.
 
-```console
+## Classic Nix
-$ nix-build -A defaultPackage.x86_64-linux
+
-```
+This section is for Nix without [flakes].
 
 To build all dependencies and start a shell in which all environment
 variables are set up so that those dependencies can be found:
@@ -35,27 +74,16 @@ variables are set up so that those dependencies can be found:
 $ nix-shell
 ```
 
-or if you have a flake-enabled nix:
+To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix develop
+$ nix-shell -A devShells.x86_64-linux.native-clang11StdenvPackages
 ```
 
-To get a shell with a different compilation environment (e.g. stdenv,
+> **Note**
-gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv):
+>
-
+> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time.
-```console
+> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`.
-$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
-```
-
-or if you have a flake-enabled nix:
-
-```console
-$ nix develop .#clang11StdenvPackages
-```
-
-Note: you can use `ccacheStdenv` to drastically improve rebuild
-time. By default, ccache keeps artifacts in `~/.cache/ccache/`.
 
 To build Nix itself in this shell:
 
@@ -71,21 +99,99 @@ To install it in `$(pwd)/outputs` and test it:
 [nix-shell]$ make install
 [nix-shell]$ make installcheck -j $NIX_BUILD_CORES
 [nix-shell]$ ./outputs/out/bin/nix --version
-nix (Nix) 3.0
+nix (Nix) 2.12
 ```
 
-If you have a flakes-enabled Nix you can replace:
+To build Nix for the current operating system and CPU architecture use
 
 ```console
-$ nix-shell
+$ nix-build
 ```
 
-by:
+You can also build Nix for one of the [supported target platforms](#target-platforms).
+
+## Platforms
+
+As specified in [`flake.nix`], Nix can be built for various platforms:
+
+- `aarch64-linux`
+- `i686-linux`
+- `x86_64-darwin`
+- `x86_64-linux`
+
+[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix
+
+In order to build Nix for a different platform than the one you're currently
+on, you need to have some way for your system Nix to build code for that
+platform. Common solutions include [remote builders] and [binfmt emulation]
+(only supported on NixOS).
+
+[remote builders]: ../advanced-topics/distributed-builds.md
+[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems
+
+These solutions let Nix perform builds as if you're on the native platform, so
+executing the build is as simple as
 
 ```console
-$ nix develop
+$ nix build .#packages.aarch64-linux.default
 ```
 
+for flake-enabled Nix, or
+
+```console
+$ nix-build -A packages.aarch64-linux.default
+```
+
+for classic Nix.
+
+You can use any of the other supported platforms in place of `aarch64-linux`.
+
+Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`.
+
+## Compilation environments
+
+Nix can be compiled using multiple environments:
+
+- `stdenv`: default;
+- `gccStdenv`: force the use of `gcc` compiler;
+- `clangStdenv`: force the use of `clang` compiler;
+- `ccacheStdenv`: enable [ccache], a compiler cache to speed up compilation.
+
+To build with one of those environments, you can use
+
+```console
+$ nix build .#nix-ccacheStdenv
+```
+
+for flake-enabled Nix, or
+
+```console
+$ nix-build -A nix-ccacheStdenv
+```
+
+for classic Nix.
+
+You can use any of the other supported environments in place of `nix-ccacheStdenv`.
+
+## Editor integration
+
+The `clangd` LSP server is installed by default on the `clang`-based `devShell`s.
+See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix).
+
+To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
+
+```console
+make clean && bear -- make -j$NIX_BUILD_CORES install
+```
+
+Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).
+
+> **Note**
+>
+> For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
+> Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
+> Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
 ## Running tests
 
 ### Unit-tests
@@ -219,7 +325,7 @@ After the CI run completes, you can check the output to extract the installer UR
 5. To generate an install command, plug this `install_url` and your GitHub username into this template:
 
     ```console
-    sh <(curl -L <install_url>) --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
+    curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
     ```
 
 <!-- #### Manually generating test installers

doc/manual/src/glossary.md

@@ -19,6 +19,13 @@
 
     [store derivation]: #gloss-store-derivation
 
+  - [instantiate]{#gloss-instantiate}, instantiation\
+    Translate a [derivation] into a [store derivation].
+
+    See [`nix-instantiate`](./command-ref/nix-instantiate.md).
+
+    [instantiate]: #gloss-instantiate
+
   - [realise]{#gloss-realise}, realisation\
     Ensure a [store path] is [valid][validity].
 

doc/manual/src/installation/env-variables.md

@@ -27,7 +27,7 @@ Set the environment variable and install Nix
 
 ```console
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh <(curl -L https://nixos.org/nix/install)
+$ curl -L https://nixos.org/nix/install | sh
 ```
 
 In the shell profile and rc files (for example, `/etc/bashrc`,

doc/manual/src/installation/installation.md

@@ -1,2 +1,38 @@
-This section describes how to install and configure Nix for first-time
+# Installation
-use.
+
+This section describes how to install and configure Nix for first-time use.
+
+The current recommended option on Linux and MacOS is [multi-user](#multi-user).
+
+## Multi-user
+
+This installation offers better sharing, improved isolation, and more security
+over a single user installation.
+
+This option requires either:
+
+* Linux running systemd, with SELinux disabled
+* MacOS
+
+```console
+$ bash <(curl -L https://nixos.org/nix/install) --daemon
+```
+
+## Single-user
+
+> Single-user is not supported on Mac.
+
+This installation has less requirements than the multi-user install, however it
+cannot offer equivalent sharing, isolation, or security.
+
+This option is suitable for systems without systemd.
+
+```console
+$ bash <(curl -L https://nixos.org/nix/install) --no-daemon
+```
+
+## Distributions
+
+The Nix community maintains installers for several distributions.
+
+They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository.

doc/manual/src/installation/installing-binary.md

@@ -3,7 +3,7 @@
 The easiest way to install Nix is to run the following command:
 
 ```console
-$ sh <(curl -L https://nixos.org/nix/install)
+$ curl -L https://nixos.org/nix/install | sh
 ```
 
 This will run the installer interactively (causing it to explain what
@@ -27,7 +27,7 @@ you can authenticate with `sudo`.
 To explicitly select a single-user installation on your system:
 
 ```console
-$ sh <(curl -L https://nixos.org/nix/install) --no-daemon
+$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
 ```
 
 This will perform a single-user installation of Nix, meaning that `/nix`
@@ -66,7 +66,7 @@ You can instruct the installer to perform a multi-user installation on
 your system:
 
 ```console
-$ sh <(curl -L https://nixos.org/nix/install) --daemon
+$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
 ```
 
 The multi-user installation of Nix will create build users between the
@@ -287,7 +287,7 @@ These install scripts can be used the same as the main NixOS.org
 installation script:
 
 ```console
-$ sh <(curl -L https://nixos.org/nix/install)
+$ curl -L https://nixos.org/nix/install | sh
 ```
 
 In the same directory of the install script are sha256 sums, and gpg

doc/manual/src/language/advanced-attributes.md

@@ -260,7 +260,7 @@ Derivations can declare some infrequently used optional attributes.
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
     attributes are serialised in JSON format and made available to the
     builder via the file `.attrs.json` in the builder’s temporary
-    directory. This obviates the need for `passAsFile` since JSON files
+    directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files
     have no size restrictions, unlike process environments.
 
     It also makes it possible to tweak derivation settings in a structured way; see
@@ -282,11 +282,13 @@ Derivations can declare some infrequently used optional attributes.
     [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
     the following attributes are available:
 
-    - `maxSize` defines the maximum size of the output path.
+    - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object).
     - `maxClosureSize` defines the maximum size of the output's closure.
     - `ignoreSelfRefs` controls whether self-references should be considered when
       checking for allowed references/requisites.
 
+    Example:
+
     ```nix
     __structuredAttrs = true;
 
@@ -305,9 +307,20 @@ Derivations can declare some infrequently used optional attributes.
     ```
 
   - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
-    When using [structured attributes](#adv-attr-structuredAttrs), the **experimental**
+    > **Warning**
-    attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`,
+    > This is an experimental feature.
-    disables scanning the build output for runtime dependencies altogether.
+    >
+    > To enable it, add the following to [nix.conf](../command-ref/conf-file.md):
+    >
+    > ```
+    > extra-experimental-features = discard-references
+    > ```
+
+    When using [structured attributes](#adv-attr-structuredAttrs), the
+    attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name.
+    If set to `true`, it disables scanning the output for runtime dependencies.
+
+    Example:
 
     ```nix
     __structuredAttrs = true;
@@ -317,5 +330,3 @@ Derivations can declare some infrequently used optional attributes.
     This is useful, for example, when generating self-contained filesystem images with
     their own embedded Nix store: hashes found inside such an image refer
     to the embedded store and not to the host's Nix store.
-
-    This is only allowed if the `discard-references` experimental feature is enabled.

doc/manual/src/language/index.md

@@ -91,7 +91,7 @@ This is an incomplete overview of language features, by example.
  <tr>
   <td>
 
-   `"hello ${ { a = "world" }.a }"`
+   `"hello ${ { a = "world"; }.a }"`
 
    `"1 2 ${toString 3}"`
 

doc/manual/src/language/operators.md

@@ -24,7 +24,7 @@
 | [Equality]                             | *expr* `==` *expr*                         | none          | 11         |
 | Inequality                             | *expr* `!=` *expr*                         | none          | 11         |
 | Logical conjunction (`AND`)            | *bool* `&&` *bool*                         | left          | 12         |
-| Logical disjunction (`OR`)             | *bool* `\|\|` *bool*                       | left          | 13         |
+| Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
 | [Logical implication]                  | *bool* `->` *bool*                         | none          | 14         |
 
 [string]: ./values.md#type-string
@@ -116,7 +116,7 @@ The result is a string.
 [store path]: ../glossary.md#gloss-store-path
 [store]: ../glossary.md#gloss-store
 
-[Path and string concatenation]: #path-and-string-concatenation
+[String and path concatenation]: #string-and-path-concatenation
 
 ## Update
 

doc/manual/src/quick-start.md

@@ -4,16 +4,16 @@ This chapter is for impatient people who don't like reading
 documentation.  For more in-depth information you are kindly referred
 to subsequent chapters.
 
-1. Install single-user Nix by running the following:
+1. Install Nix by running the following:
 
    ```console
-   $ bash <(curl -L https://nixos.org/nix/install)
+   $ curl -L https://nixos.org/nix/install | sh
    ```
 
-   This will install Nix in `/nix`. The install script will create
+   The install script will use `sudo`, so make sure you have sufficient rights.
-   `/nix` using `sudo`, so make sure you have sufficient rights.  (For
+   On Linux, `--daemon` can be omitted for a single-user install.
-   other installation methods, see
+
-   [here](installation/installation.md).)
+   For other installation methods, see [here](installation/installation.md).
 
 1. See what installable packages are currently available in the
    channel:

doc/manual/src/release-notes/rl-2.14.md

@@ -0,0 +1,22 @@
+# Release 2.14 (2023-02-28)
+
+* A new function `builtins.readFileType` is available. It is similar to
+  `builtins.readDir` but acts on a single file or directory.
+
+* In flakes, the `.outPath` attribute of a flake now always refers to
+  the directory containing the `flake.nix`. This was not the case for
+  when `flake.nix` was in a subdirectory of e.g. a Git repository.
+  The root of the source of a flake in a subdirectory is still
+  available in `.sourceInfo.outPath`.
+
+* In derivations that use structured attributes, you can now use `unsafeDiscardReferences`
+  to disable scanning a given output for runtime dependencies:
+  ```nix
+  __structuredAttrs = true;
+  unsafeDiscardReferences.out = true;
+  ```
+  This is useful e.g. when generating self-contained filesystem images with
+  their own embedded Nix store: hashes found inside such an image refer
+  to the embedded store and not to the host's Nix store.
+
+  This requires the `discard-references` experimental feature.

doc/manual/src/release-notes/rl-next.md

@@ -1,22 +1,2 @@
 # Release X.Y (202?-??-??)
 
-* A new function `builtins.readFileType` is available. It is similar to
-  `builtins.readDir` but acts on a single file or directory.
-
-* The `builtins.readDir` function has been optimized when encountering not-yet-known
-  file types from POSIX's `readdir`. In such cases the type of each file is/was
-  discovered by making multiple syscalls. This change makes these operations
-  lazy such that these lookups will only be performed if the attribute is used.
-  This optimization affects a minority of filesystems and operating systems.
-
-* In derivations that use structured attributes, you can now use `unsafeDiscardReferences`
-  to disable scanning a given output for runtime dependencies:
-  ```nix
-  __structuredAttrs = true;
-  unsafeDiscardReferences.out = true;
-  ```
-  This is useful e.g. when generating self-contained filesystem images with
-  their own embedded Nix store: hashes found inside such an image refer
-  to the embedded store and not to the host's Nix store.
-
-  This requires the `discard-references` experimental feature.

flake.nix

@@ -8,10 +8,11 @@
   outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }:
 
     let
+      inherit (nixpkgs) lib;
 
       officialRelease = false;
 
-      version = nixpkgs.lib.fileContents ./.version + versionSuffix;
+      version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
         then ""
@@ -25,36 +26,42 @@
 
       stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
 
-      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      forAllSystems = lib.genAttrs systems;
-      forAllSystemsAndStdenvs = f: forAllSystems (system:
+
-        nixpkgs.lib.listToAttrs
+      forAllCrossSystems = lib.genAttrs crossSystems;
-          (map
+
-            (n:
+      forAllStdenvs = f:
-            nixpkgs.lib.nameValuePair "${n}Packages" (
+        lib.listToAttrs
-              f system n
+          (map
-            )) stdenvs
+            (stdenvName: {
-          )
+              name = "${stdenvName}Packages";
-      );
+              value = f stdenvName;
+            })
+            stdenvs);
 
-      forAllStdenvs = f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor =
+      nixpkgsFor = forAllSystems
-        let stdenvsPackages = forAllSystemsAndStdenvs
+        (system: let
-          (system: stdenv:
+          make-pkgs = crossSystem: stdenv: import nixpkgs {
-            import nixpkgs {
+            inherit system crossSystem;
-              inherit system;
             overlays = [
               (overlayFor (p: p.${stdenv}))
             ];
-            }
+          };
-          );
+          stdenvs = forAllStdenvs (make-pkgs null);
-        in
+          native = stdenvs.stdenvPackages;
-        # Add the `stdenvPackages` at toplevel, both because these are the ones
+        in {
-        # we want most of the time and for backwards compatibility
+          inherit stdenvs native;
-        forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
+          static = native.pkgsStatic;
+          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+        });
 
-      commonDeps = { pkgs, isStatic ? false }: with pkgs; rec {
+      commonDeps =
+        { pkgs
+        , isStatic ? pkgs.stdenv.hostPlatform.isStatic
+        }:
+        with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
         # if not (legacy) fallback and hope it's sufficient.
         sh = pkgs.busybox-sandbox-shell or (busybox.override {
@@ -144,7 +151,7 @@
       };
 
       installScriptFor = systems:
-        with nixpkgsFor.x86_64-linux;
+        with nixpkgsFor.x86_64-linux.native;
         runCommand "installer-script"
           { buildInputs = [ nix ];
           }
@@ -208,8 +215,9 @@
         installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
       };
 
-      binaryTarball = buildPackages: nix: pkgs:
+      binaryTarball = nix: pkgs:
         let
+          inherit (pkgs) buildPackages;
           inherit (pkgs) cacert;
           installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
         in
@@ -289,7 +297,15 @@
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
-          nix = with final; with commonDeps { inherit pkgs; }; currentStdenv.mkDerivation {
+          nix =
+          with final;
+          with commonDeps {
+            inherit pkgs;
+            inherit (currentStdenv.hostPlatform) isStatic;
+          };
+          let
+            canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
+          in currentStdenv.mkDerivation {
             name = "nix-${version}";
             inherit version;
 
@@ -300,24 +316,26 @@
             outputs = [ "out" "dev" "doc" ];
 
             nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ awsDeps;
+            buildInputs = buildDeps
+              # There have been issues building these dependencies
+              ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps;
 
             propagatedBuildInputs = propagatedDeps;
 
             disallowedReferences = [ boost ];
 
-            preConfigure =
+            preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
                 cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
                 rm -f $out/lib/*.a
-                ${lib.optionalString currentStdenv.isLinux ''
+                ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
                   patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
                 ''}
-                ${lib.optionalString currentStdenv.isDarwin ''
+                ${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
@@ -328,7 +346,9 @@
               '';
 
             configureFlags = configureFlags ++
-              [ "--sysconfdir=/etc" ];
+              [ "--sysconfdir=/etc" ] ++
+              lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
+              lib.optional (!canRunInstalled) "--disable-doc-gen";
 
             enableParallelBuilding = true;
 
@@ -352,10 +372,12 @@
             doInstallCheck = true;
             installCheckFlags = "sysconfdir=$(out)/etc";
 
-            separateDebugInfo = true;
+            separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
 
             strictDeps = true;
 
+            hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
             passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation {
               name = "nix-perl-${version}";
 
@@ -388,7 +410,7 @@
               postUnpack = "sourceRoot=$sourceRoot/perl";
             });
 
-            meta.platforms = systems;
+            meta.platforms = lib.platforms.unix;
           };
 
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
@@ -414,14 +436,15 @@
       # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
       runNixOSTestFor = system: test: nixos-lib.runTest {
         imports = [ test ];
-        hostPkgs = nixpkgsFor.${system};
+        hostPkgs = nixpkgsFor.${system}.native;
         defaults = {
-          nixpkgs.pkgs = nixpkgsFor.${system};
+          nixpkgs.pkgs = nixpkgsFor.${system}.native;
         };
         _module.args.nixpkgs = nixpkgs;
       };
 
     in {
+      inherit nixpkgsFor;
 
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
@@ -430,32 +453,28 @@
       hydraJobs = {
 
         # Binary package for various platforms.
-        build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
+        build = forAllSystems (system: self.packages.${system}.nix);
 
-        buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
+        buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
-        buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem:
+        buildCross = forAllCrossSystems (crossSystem:
-          nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
+          lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
-        buildNoGc = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
+        buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
 
         # Perl bindings for various platforms.
-        perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
+        perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system});
+        binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
 
-        binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: {
+        binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system:
-          name = crossSystem;
+          forAllCrossSystems (crossSystem:
-          value = let
+            binaryTarball
-            nixpkgsCross = import nixpkgs {
+              self.packages.${system}."nix-${crossSystem}"
-              inherit system crossSystem;
+              nixpkgsFor.${system}.cross.${crossSystem}));
-              overlays = [ self.overlays.default ];
-            };
-          in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross;
-        }) crossSystems));
 
         # The first half of the installation script. This is uploaded
         # to https://nixos.org/nix/install. It downloads the binary
@@ -465,11 +484,11 @@
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # docker image with Nix inside
-        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
+        dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
         # Line coverage analysis.
         coverage =
-          with nixpkgsFor.x86_64-linux;
+          with nixpkgsFor.x86_64-linux.native;
           with commonDeps { inherit pkgs; };
 
           releaseTools.coverageAnalysis {
@@ -499,6 +518,8 @@
           };
 
         # System tests.
+        tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix;
+
         tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix;
 
         tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix;
@@ -511,7 +532,7 @@
 
         tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix;
 
-        tests.setuid = nixpkgs.lib.genAttrs
+        tests.setuid = lib.genAttrs
           ["i686-linux" "x86_64-linux"]
           (system: runNixOSTestFor system ./tests/nixos/setuid.nix);
 
@@ -519,7 +540,7 @@
         # Make sure that nix-env still produces the exact same result
         # on a particular version of Nixpkgs.
         tests.evalNixpkgs =
-          with nixpkgsFor.x86_64-linux;
+          with nixpkgsFor.x86_64-linux.native;
           runCommand "eval-nixos" { buildInputs = [ nix ]; }
             ''
               type -p nix-env
@@ -530,18 +551,18 @@
             '';
 
         tests.nixpkgsLibTests =
-          nixpkgs.lib.genAttrs systems (system:
+          forAllSystems (system:
             import (nixpkgs + "/lib/tests/release.nix")
-              { pkgs = nixpkgsFor.${system}; }
+              { pkgs = nixpkgsFor.${system}.native; }
           );
 
         metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
-          pkgs = nixpkgsFor.x86_64-linux;
+          pkgs = nixpkgsFor.x86_64-linux.native;
           nixpkgs = nixpkgs-regression;
         };
 
         installTests = forAllSystems (system:
-          let pkgs = nixpkgsFor.${system}; in
+          let pkgs = nixpkgsFor.${system}.native; in
           pkgs.runCommand "install-tests" {
             againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
             againstCurrentUnstable =
@@ -566,67 +587,18 @@
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
-      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
+      } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });
 
       packages = forAllSystems (system: rec {
-        inherit (nixpkgsFor.${system}) nix;
+        inherit (nixpkgsFor.${system}.native) nix;
         default = nix;
-      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
+      } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
-        nix-static = let
+        nix-static = nixpkgsFor.${system}.static.nix;
-          nixpkgs = nixpkgsFor.${system}.pkgsStatic;
-        in with commonDeps { pkgs = nixpkgs; isStatic = true; }; nixpkgs.stdenv.mkDerivation {
-          name = "nix-${version}";
-
-          src = self;
-
-          VERSION_SUFFIX = versionSuffix;
-
-          outputs = [ "out" "dev" "doc" ];
-
-          nativeBuildInputs = nativeBuildDeps;
-          buildInputs = buildDeps ++ propagatedDeps;
-
-          # Work around pkgsStatic disabling all tests.
-          # Remove in NixOS 22.11, see https://github.com/NixOS/nixpkgs/pull/140271.
-          preHook =
-            ''
-              doCheck=1
-              doInstallCheck=1
-            '';
-
-          configureFlags =
-            configureFlags ++
-            [ "--sysconfdir=/etc"
-              "--enable-embedded-sandbox-shell"
-            ];
-
-          enableParallelBuilding = true;
-
-          makeFlags = "profiledir=$(out)/etc/profile.d";
-
-          installFlags = "sysconfdir=$(out)/etc";
-
-          postInstall = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-            mkdir -p $out/nix-support
-            echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-          '';
-
-          installCheckFlags = "sysconfdir=$(out)/etc";
-
-          stripAllList = ["bin"];
-
-          strictDeps = true;
-
-          hardeningDisable = [ "pie" ];
-        };
-
         dockerImage =
           let
-            pkgs = nixpkgsFor.${system};
+            pkgs = nixpkgsFor.${system}.native;
             image = import ./docker.nix { inherit pkgs; tag = version; };
           in
           pkgs.runCommand
@@ -638,70 +610,30 @@
               ln -s ${image} $image
               echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
             '';
-      }
+      } // builtins.listToAttrs (map
-
+          (crossSystem: {
-      // builtins.listToAttrs (map (crossSystem: {
             name = "nix-${crossSystem}";
-        value = let
+            value = nixpkgsFor.${system}.cross.${crossSystem}.nix;
-          nixpkgsCross = import nixpkgs {
+          })
-            inherit system crossSystem;
+          crossSystems)
-            overlays = [ self.overlays.default ];
+        // builtins.listToAttrs (map
-          };
+          (stdenvName: {
-          inherit (nixpkgsCross) lib;
+            name = "nix-${stdenvName}";
-        in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation {
+            value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix;
-          name = "nix-${version}";
+          })
+          stdenvs)));
 
-          src = self;
+      devShells = let
-
+        makeShell = pkgs: stdenv:
-          VERSION_SUFFIX = versionSuffix;
-
-          outputs = [ "out" "dev" "doc" ];
-
-          nativeBuildInputs = nativeBuildDeps;
-          buildInputs = buildDeps ++ propagatedDeps;
-
-          configureFlags = [
-            "CXXFLAGS=-I${lib.getDev nixpkgsCross.rapidcheck}/extras/gtest/include"
-            "--sysconfdir=/etc"
-            "--disable-doc-gen"
-          ];
-
-          enableParallelBuilding = true;
-
-          makeFlags = "profiledir=$(out)/etc/profile.d";
-
-          doCheck = true;
-
-          installFlags = "sysconfdir=$(out)/etc";
-
-          postInstall = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-            mkdir -p $out/nix-support
-            echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-          '';
-
-          doInstallCheck = true;
-          installCheckFlags = "sysconfdir=$(out)/etc";
-        };
-      }) (if system == "x86_64-linux" then crossSystems else [])))
-
-      // (builtins.listToAttrs (map (stdenvName:
-        nixpkgsFor.${system}.lib.nameValuePair
-          "nix-${stdenvName}"
-          nixpkgsFor.${system}."${stdenvName}Packages".nix
-      ) stdenvs)));
-
-      devShells = forAllSystems (system:
-        forAllStdenvs (stdenv:
-          with nixpkgsFor.${system};
           with commonDeps { inherit pkgs; };
-          nixpkgsFor.${system}.${stdenv}.mkDerivation {
+          stdenv.mkDerivation {
             name = "nix";
 
             outputs = [ "out" "dev" "doc" ];
 
-            nativeBuildInputs = nativeBuildDeps;
+            nativeBuildInputs = nativeBuildDeps
+                                ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]);
+
             buildInputs = buildDeps ++ propagatedDeps ++ awsDeps;
 
             inherit configureFlags;
@@ -719,10 +651,21 @@
                 # Make bash completion work.
                 XDG_DATA_DIRS+=:$out/share
               '';
+          };
+        in
+        forAllSystems (system:
+          let
+            makeShells = prefix: pkgs:
+              lib.mapAttrs'
+              (k: v: lib.nameValuePair "${prefix}-${k}" v)
+              (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName}));
+          in
+            (makeShells "native" nixpkgsFor.${system}.native) //
+            (makeShells "static" nixpkgsFor.${system}.static) //
+            (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            {
+              default = self.devShells.${system}.native-stdenvPackages;
             }
-        )
-        // { default = self.devShells.${system}.stdenv; }
         );
-
   };
 }

maintainers/backporting.md

@@ -0,0 +1,12 @@
+
+# Backporting
+
+To [automatically backport a pull request](https://github.com/NixOS/nix/blob/master/.github/workflows/backport.yml) to a release branch once it's merged, assign it a label of the form [`backport <branch>`](https://github.com/NixOS/nix/labels?q=backport).
+
+Since [GitHub Actions workflows will not trigger other workflows](https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow), checks on the automatic backport need to be triggered by another actor.
+This is achieved by closing and reopening the backport pull request.
+
+This specifically affects the [`installer_test`] check.
+Note that it only runs after the other tests, so it may take a while to appear.
+
+[`installer_test`]: https://github.com/NixOS/nix/blob/895dfc656a21f6252ddf48df0d1f215effa04ecb/.github/workflows/ci.yml#L70-L91

maintainers/release-process.md

@@ -0,0 +1,181 @@
+# Nix release process
+
+## Release artifacts
+
+The release process is intended to create the following for each
+release:
+
+* A Git tag
+
+* Binary tarballs in https://releases.nixos.org/?prefix=nix/
+
+* Docker images
+
+* Closures in https://cache.nixos.org
+
+* (Optionally) Updated `fallback-paths.nix` in Nixpkgs
+
+* An updated manual on https://nixos.org/manual/nix/stable/
+
+## Creating a new release from the `master` branch
+
+* Make sure that the [Hydra `master` jobset](https://hydra.nixos.org/jobset/nix/master) succeeds.
+
+* In a checkout of the Nix repo, make sure you're on `master` and run
+  `git pull`.
+
+* Move the contents of `doc/manual/src/release-notes/rl-next.md`
+  (except the first line) to
+  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
+  the contents of `.version` *without* the patch level, e.g. `2.12`
+  rather than `2.12.0`).
+
+* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
+
+  ```
+  # Release 2.12 (2022-12-06)
+  ```
+
+* Proof-read / edit / rearrange the release notes. Breaking changes
+  and highlights should go to the top.
+
+* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
+  (*not* `SUMMARY.md`), e.g.
+
+  ```
+  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
+  ```
+
+* Run
+
+  ```console
+  $ git checkout -b release-notes
+  $ git add doc/manual/src/release-notes/rl-$VERSION.md
+  $ git commit -a -m 'Release notes'
+  $ git push --set-upstream $REMOTE release-notes
+  ```
+
+* Create a PR for `release-notes`.
+
+* Wait for the PR to be merged.
+
+* Create a branch for the release:
+
+  ```console
+  $ git checkout master
+  $ git pull
+  $ git checkout -b $VERSION-maintenance
+  ```
+
+* Mark the release as stable:
+
+  ```console
+  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  ```
+
+  This removes the link to `rl-next.md` from the manual and sets
+  `officialRelease = true` in `flake.nix`.
+
+* Push the release branch:
+
+  ```console
+  $ git push --set-upstream origin $VERSION-maintenance
+  ```
+
+* Create a jobset for the release branch on Hydra as follows:
+
+  * Go to the jobset of the previous release
+  (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11).
+
+  * Select `Actions -> Clone this jobset`.
+
+  * Set identifier to `maintenance-$VERSION`.
+
+  * Set description to `$VERSION release branch`.
+
+  * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`.
+
+  * Hit `Create jobset`.
+
+* Wait for the new jobset to evaluate and build. If impatient, go to
+  the evaluation and select `Actions -> Bump builds to front of
+  queue`.
+
+* When the jobset evaluation has succeeded building, take note of the
+  evaluation ID (e.g. `1780832` in
+  `https://hydra.nixos.org/eval/1780832`).
+
+* Tag the release and upload the release artifacts to
+  [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
+
+  ```console
+  $ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
+  ```
+
+  Note: `IS_LATEST=1` causes the `latest-release` branch to be
+  force-updated. This is used by the `nixos.org` website to get the
+  [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
+
+  TODO: This script requires the right AWS credentials. Document.
+
+  TODO: This script currently requires a
+  `/home/eelco/Dev/nix-pristine` and
+  `/home/eelco/Dev/nixpkgs-pristine`.
+
+  TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
+* Prepare for the next point release by editing `.version` to
+  e.g.
+
+  ```console
+  $ echo 2.12.1 > .version
+  $ git commit -a -m 'Bump version'
+  $ git push
+  ```
+
+  Commit and push this to the maintenance branch.
+
+* Bump the version of `master`:
+
+  ```console
+  $ git checkout master
+  $ git pull
+  $ NEW_VERSION=2.13.0
+  $ echo -n $NEW_VERSION > .version
+  $ git checkout -b bump-$NEW_VERSION
+  $ git commit -a -m 'Bump version'
+  $ git push --set-upstream origin bump-$NEW_VERSION
+  ```
+
+  Make a pull request and auto-merge it.
+
+* Create a milestone for the next release, move all unresolved issues
+  from the previous milestone, and close the previous milestone. Set
+  the date for the next milestone 6 weeks from now.
+
+* Create a backport label
+
+* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of
+  `rl-$VERSION.md`.
+
+## Creating a point release
+
+* Wait for the desired evaluation of the maintenance jobset to finish
+  building.
+
+* Run
+
+  ```console
+  $ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
+  ```
+
+  Omit `IS_LATEST=1` when creating a point release that is not on the
+  most recent stable branch. This prevents `nixos.org` to going back
+  to an older release.
+
+* Bump the version number of the release branch as above (e.g. to
+  `2.12.2`).
+  
+## Recovering from mistakes
+
+`upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release.
+

mk/libraries.mk

@@ -67,6 +67,7 @@ define build-library
 
   $(1)_LDFLAGS_USE :=
   $(1)_LDFLAGS_USE_INSTALLED :=
+  $(1)_LIB_CLOSURE := $(1)
 
   $$(eval $$(call create-dir, $$(_d)))
 
@@ -128,10 +129,12 @@ define build-library
 	+$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$^
 	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
-    $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
+    $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
     $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a
 
+    $(1)_LIB_CLOSURE += $$($(1)_LIBS)
+
   endif
 
   $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED)

mk/programs.mk

@@ -30,7 +30,7 @@ define build-program
   _d := $(buildprefix)$$($(1)_DIR)
   _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
-  _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
+  _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH)))
   $(1)_PATH := $$(_d)/$$($(1)_NAME)
 
   $$(eval $$(call create-dir, $$(_d)))
@@ -58,7 +58,7 @@ define build-program
     else
 
       $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
+	+$$(trace-install) install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
     endif
   endif

perl/Makefile

@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -std=c++20 -I ../src
+GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src
 
 -include Makefile.config
 

perl/lib/Nix/Store.xs

@@ -26,6 +26,7 @@ static ref<Store> store()
     static std::shared_ptr<Store> _store;
     if (!_store) {
         try {
+            initLibStore();
             loadConfFile();
             settings.lockCPU = false;
             _store = openStore();

scripts/install-multi-user.sh

@@ -136,7 +136,7 @@ EOF
     cat <<EOF
 $step. Delete the files Nix added to your system:
 
-  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
+  sudo rm -rf "/etc/nix" "$NIX_ROOT" "$ROOT_HOME/.nix-profile" "$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.local/state/nix" "$ROOT_HOME/.cache/nix" "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" "$HOME/.local/state/nix" "$HOME/.cache/nix"
 
 and that is it.
 

scripts/install-nix-from-closure.sh

@@ -188,6 +188,8 @@ fi
 # shellcheck source=./nix-profile.sh.in
 . "$nix/etc/profile.d/nix.sh"
 
+NIX_LINK="$HOME/.nix-profile"
+
 if ! "$nix/bin/nix-env" -i "$nix"; then
     echo "$0: unable to install Nix into your default profile" >&2
     exit 1
@@ -196,7 +198,7 @@ fi
 # Install an SSL certificate bundle.
 if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
     "$nix/bin/nix-env" -i "$cacert"
-    export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt"
+    export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ssl/certs/ca-bundle.crt"
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
@@ -214,8 +216,8 @@ fi
 
 added=
 p=
-p_sh=$HOME/.nix-profile/etc/profile.d/nix.sh
+p_sh=$NIX_LINK/etc/profile.d/nix.sh
-p_fish=$HOME/.nix-profile/etc/profile.d/nix.fish
+p_fish=$NIX_LINK/etc/profile.d/nix.fish
 if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
     # Make the shell source nix.sh during login.
     for i in .bash_profile .bash_login .profile; do

scripts/nix-profile-daemon.sh.in

@@ -2,7 +2,33 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
+NIX_LINK=$HOME/.nix-profile
+if [ -n "$XDG_STATE_HOME" ]; then
+    NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile"
+else
+    NIX_LINK_NEW=$HOME/.local/state/nix/profile
+fi
+if ! [ -e "$NIX_LINK" ]; then
+    NIX_LINK="$NIX_LINK_NEW"
+else
+    if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
+        warning="\033[1;35mwarning:\033[0m"
+        printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+        if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
+            printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+        else
+            # This should be an exceptionally rare occasion: the only way to get it would be to
+            # 1. Update to newer Nix;
+            # 2. Remove .nix-profile;
+            # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+            # 4. Roll back to older Nix.
+            # If someone did all that, they can probably figure out how to migrate the profile.
+            printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+        fi
+    fi
+fi
+
+export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
@@ -34,4 +60,5 @@ else
   unset -f check_nix_profiles
 fi
 
-export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
+export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
+unset NIX_LINK

scripts/nix-profile.sh.in

@@ -2,11 +2,35 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     # Set up the per-user profile.
 
-    NIX_LINK=$HOME/.nix-profile
+    NIX_LINK="$HOME/.nix-profile"
+    if [ -n "$XDG_STATE_HOME" ]; then
+        NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile"
+    else
+        NIX_LINK_NEW="$HOME/.local/state/nix/profile"
+    fi
+    if ! [ -e "$NIX_LINK" ]; then
+        NIX_LINK="$NIX_LINK_NEW"
+    else
+        if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
+            warning="\033[1;35mwarning:\033[0m"
+            printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
+            if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
+                printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
+            else
+                # This should be an exceptionally rare occasion: the only way to get it would be to
+                # 1. Update to newer Nix;
+                # 2. Remove .nix-profile;
+                # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
+                # 4. Roll back to older Nix.
+                # If someone did all that, they can probably figure out how to migrate the profile.
+                printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
+            fi
+        fi
+    fi
 
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
-    export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
+    export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
@@ -31,5 +55,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     fi
 
     export PATH="$NIX_LINK/bin:$PATH"
-    unset NIX_LINK
+    unset NIX_LINK NIX_LINK_NEW
 fi

src/build-remote/build-remote.cc

@@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv)
             settings.set(name, value);
         }
 
+        auto maxBuildJobs = settings.maxBuildJobs;
         settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
         initPlugins();
@@ -112,10 +113,14 @@ static int main_build_remote(int argc, char * * argv)
             drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling
+            /* It would be possible to build locally after some builds clear out,
+               so don't show the warning now: */
+            bool couldBuildLocally = maxBuildJobs > 0
                  &&  (  neededSystem == settings.thisSystem
                      || settings.extraPlatforms.get().count(neededSystem) > 0)
                  &&  allSupportedLocally(*store, requiredFeatures);
+            /* It's possible to build this locally right now: */
+            bool canBuildLocally = amWilling && couldBuildLocally;
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -214,7 +219,7 @@ static int main_build_remote(int argc, char * * argv)
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
 
-                        printMsg(canBuildLocally ? lvlChatty : lvlWarn, error);
+                        printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error);
 
                         std::cerr << "# decline\n";
                     }

src/libcmd/command.cc

@@ -4,6 +4,7 @@
 #include "derivations.hh"
 #include "nixexpr.hh"
 #include "profiles.hh"
+#include "repl.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -121,12 +122,22 @@ ref<EvalState> EvalCommand::getEvalState()
             ;
 
         if (startReplOnEvalErrors) {
-            evalState->debugRepl = &runRepl;
+            evalState->debugRepl = &AbstractNixRepl::runSimple;
         };
     }
     return ref<EvalState>(evalState);
 }
 
+MixOperateOnOptions::MixOperateOnOptions()
+{
+    addFlag({
+        .longName = "derivation",
+        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
+        .category = installablesCategory,
+        .handler = {&operateOn, OperateOn::Derivation},
+    });
+}
+
 BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
@@ -208,20 +219,6 @@ void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePath
     run(store, *storePaths.begin());
 }
 
-Strings editorFor(const Path & file, uint32_t line)
-{
-    auto editor = getEnv("EDITOR").value_or("cat");
-    auto args = tokenizeString<Strings>(editor);
-    if (line > 0 && (
-        editor.find("emacs") != std::string::npos ||
-        editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos ||
-        editor.find("kak") != std::string::npos))
-        args.push_back(fmt("+%d", line));
-    args.push_back(file);
-    return args;
-}
-
 MixProfile::MixProfile()
 {
     addFlag({

src/libcmd/command.hh

@@ -94,12 +94,8 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
 {
     std::optional<Path> file;
     std::optional<std::string> expr;
-    bool readOnlyMode = false;
 
-    // FIXME: move this; not all commands (e.g. 'nix run') use it.
+    SourceExprCommand();
-    OperateOn operateOn = OperateOn::Output;
-
-    SourceExprCommand(bool supportReadOnlyMode = false);
 
     std::vector<std::shared_ptr<Installable>> parseInstallables(
         ref<Store> store, std::vector<std::string> ss);
@@ -114,6 +110,11 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
     void completeInstallable(std::string_view prefix);
 };
 
+struct MixReadOnlyOption : virtual Args
+{
+    MixReadOnlyOption();
+};
+
 /* A command that operates on a list of "installables", which can be
    store paths, attribute paths, Nix expressions, etc. */
 struct InstallablesCommand : virtual Args, SourceExprCommand
@@ -139,7 +140,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand
 {
     std::shared_ptr<Installable> installable;
 
-    InstallableCommand(bool supportReadOnlyMode = false);
+    InstallableCommand();
 
     void prepare() override;
 
@@ -153,8 +154,15 @@ private:
     std::string _installable{"."};
 };
 
+struct MixOperateOnOptions : virtual Args
+{
+    OperateOn operateOn = OperateOn::Output;
+
+    MixOperateOnOptions();
+};
+
 /* A command that operates on zero or more store paths. */
-struct BuiltPathsCommand : public InstallablesCommand
+struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions
 {
 private:
 
@@ -227,10 +235,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-/* Helper function to generate args that invoke $EDITOR on
-   filename:lineno. */
-Strings editorFor(const Path & file, uint32_t line);
-
 struct MixProfile : virtual StoreCommand
 {
     std::optional<Path> profile;
@@ -280,8 +284,4 @@ void printClosureDiff(
     const StorePath & afterPath,
     std::string_view indent);
 
-
-void runRepl(
-    ref<EvalState> evalState,
-    const ValMap & extraEnv);
 }

src/libcmd/editor-for.cc

@@ -0,0 +1,20 @@
+#include "util.hh"
+#include "editor-for.hh"
+
+namespace nix {
+
+Strings editorFor(const Path & file, uint32_t line)
+{
+    auto editor = getEnv("EDITOR").value_or("cat");
+    auto args = tokenizeString<Strings>(editor);
+    if (line > 0 && (
+        editor.find("emacs") != std::string::npos ||
+        editor.find("nano") != std::string::npos ||
+        editor.find("vim") != std::string::npos ||
+        editor.find("kak") != std::string::npos))
+        args.push_back(fmt("+%d", line));
+    args.push_back(file);
+    return args;
+}
+
+}

src/libcmd/editor-for.hh

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+/* Helper function to generate args that invoke $EDITOR on
+   filename:lineno. */
+Strings editorFor(const Path & file, uint32_t line);
+
+}

src/libcmd/installable-attr-path.cc

@@ -0,0 +1,109 @@
+#include "globals.hh"
+#include "installable-attr-path.hh"
+#include "outputs-spec.hh"
+#include "util.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "flake/flake.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+InstallableAttrPath::InstallableAttrPath(
+    ref<EvalState> state,
+    SourceExprCommand & cmd,
+    Value * v,
+    const std::string & attrPath,
+    ExtendedOutputsSpec extendedOutputsSpec)
+    : InstallableValue(state)
+    , cmd(cmd)
+    , v(allocRootValue(v))
+    , attrPath(attrPath)
+    , extendedOutputsSpec(std::move(extendedOutputsSpec))
+{ }
+
+std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
+{
+    auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
+    state.forceValue(*vRes, pos);
+    return {vRes, pos};
+}
+
+DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
+{
+    auto v = toValue(*state).first;
+
+    Bindings & autoArgs = *cmd.getAutoArgs(*state);
+
+    DrvInfos drvInfos;
+    getDerivations(*state, *v, "", autoArgs, drvInfos, false);
+
+    // Backward compatibility hack: group results by drvPath. This
+    // helps keep .all output together.
+    std::map<StorePath, OutputsSpec> byDrvPath;
+
+    for (auto & drvInfo : drvInfos) {
+        auto drvPath = drvInfo.queryDrvPath();
+        if (!drvPath)
+            throw Error("'%s' is not a derivation", what());
+
+        auto newOutputs = std::visit(overloaded {
+            [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
+                std::set<std::string> outputsToInstall;
+                for (auto & output : drvInfo.queryOutputs(false, true))
+                    outputsToInstall.insert(output.first);
+                return OutputsSpec::Names { std::move(outputsToInstall) };
+            },
+            [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
+                return e;
+            },
+        }, extendedOutputsSpec.raw());
+
+        auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
+
+        if (!didInsert)
+            iter->second = iter->second.union_(newOutputs);
+    }
+
+    DerivedPathsWithInfo res;
+    for (auto & [drvPath, outputs] : byDrvPath)
+        res.push_back({
+            .path = DerivedPath::Built {
+                .drvPath = drvPath,
+                .outputs = outputs,
+            },
+        });
+
+    return res;
+}
+
+InstallableAttrPath InstallableAttrPath::parse(
+    ref<EvalState> state,
+    SourceExprCommand & cmd,
+    Value * v,
+    std::string_view prefix,
+    ExtendedOutputsSpec extendedOutputsSpec)
+{
+    return {
+        state, cmd, v,
+        prefix == "." ? "" : std::string { prefix },
+        extendedOutputsSpec
+    };
+}
+
+}

src/libcmd/installable-attr-path.hh

@@ -0,0 +1,56 @@
+#include "globals.hh"
+#include "installable-value.hh"
+#include "outputs-spec.hh"
+#include "util.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+class InstallableAttrPath : public InstallableValue
+{
+    SourceExprCommand & cmd;
+    RootValue v;
+    std::string attrPath;
+    ExtendedOutputsSpec extendedOutputsSpec;
+
+    InstallableAttrPath(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        const std::string & attrPath,
+        ExtendedOutputsSpec extendedOutputsSpec);
+
+    std::string what() const override { return attrPath; };
+
+    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+public:
+
+    static InstallableAttrPath parse(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        std::string_view prefix,
+        ExtendedOutputsSpec extendedOutputsSpec);
+};
+
+}

src/libcmd/installable-derived-path.cc

@@ -0,0 +1,70 @@
+#include "installable-derived-path.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+std::string InstallableDerivedPath::what() const
+{
+    return derivedPath.to_string(*store);
+}
+
+DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
+{
+    return {{.path = derivedPath, .info = {} }};
+}
+
+std::optional<StorePath> InstallableDerivedPath::getStorePath()
+{
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & bfd) {
+            return bfd.drvPath;
+        },
+        [&](const DerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+    }, derivedPath.raw());
+}
+
+InstallableDerivedPath InstallableDerivedPath::parse(
+    ref<Store> store,
+    std::string_view prefix,
+    ExtendedOutputsSpec extendedOutputsSpec)
+{
+    auto derivedPath = std::visit(overloaded {
+        // If the user did not use ^, we treat the output more liberally.
+        [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
+            // First, we accept a symlink chain or an actual store path.
+            auto storePath = store->followLinksToStorePath(prefix);
+            // Second, we see if the store path ends in `.drv` to decide what sort
+            // of derived path they want.
+            //
+            // This handling predates the `^` syntax. The `^*` in
+            // `/nix/store/hash-foo.drv^*` unambiguously means "do the
+            // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could
+            // also unambiguously mean "do the DerivedPath::Opaque` case".
+            //
+            // Issue #7261 tracks reconsidering this `.drv` dispatching.
+            return storePath.isDerivation()
+                ? (DerivedPath) DerivedPath::Built {
+                    .drvPath = std::move(storePath),
+                    .outputs = OutputsSpec::All {},
+                }
+                : (DerivedPath) DerivedPath::Opaque {
+                    .path = std::move(storePath),
+                };
+        },
+        // If the user did use ^, we just do exactly what is written.
+        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+            return DerivedPath::Built {
+                .drvPath = store->parseStorePath(prefix),
+                .outputs = outputSpec,
+            };
+        },
+    }, extendedOutputsSpec.raw());
+    return InstallableDerivedPath {
+        store,
+        std::move(derivedPath),
+    };
+}
+
+}

src/libcmd/installable-derived-path.hh

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "installables.hh"
+
+namespace nix {
+
+struct InstallableDerivedPath : Installable
+{
+    ref<Store> store;
+    DerivedPath derivedPath;
+
+    InstallableDerivedPath(ref<Store> store, DerivedPath && derivedPath)
+        : store(store), derivedPath(std::move(derivedPath))
+    { }
+
+    std::string what() const override;
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+    std::optional<StorePath> getStorePath() override;
+
+    static InstallableDerivedPath parse(
+        ref<Store> store,
+        std::string_view prefix,
+        ExtendedOutputsSpec extendedOutputsSpec);
+};
+
+}

src/libcmd/installable-flake.cc

@@ -0,0 +1,236 @@
+#include "globals.hh"
+#include "installable-flake.hh"
+#include "installable-derived-path.hh"
+#include "outputs-spec.hh"
+#include "util.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "flake/flake.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+std::vector<std::string> InstallableFlake::getActualAttrPaths()
+{
+    std::vector<std::string> res;
+
+    for (auto & prefix : prefixes)
+        res.push_back(prefix + *attrPaths.begin());
+
+    for (auto & s : attrPaths)
+        res.push_back(s);
+
+    return res;
+}
+
+Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
+{
+    auto vFlake = state.allocValue();
+
+    callFlake(state, lockedFlake, *vFlake);
+
+    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    assert(aOutputs);
+
+    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
+
+    return aOutputs->value;
+}
+
+static std::string showAttrPaths(const std::vector<std::string> & paths)
+{
+    std::string s;
+    for (const auto & [n, i] : enumerate(paths)) {
+        if (n > 0) s += n + 1 == paths.size() ? " or " : ", ";
+        s += '\''; s += i; s += '\'';
+    }
+    return s;
+}
+
+InstallableFlake::InstallableFlake(
+    SourceExprCommand * cmd,
+    ref<EvalState> state,
+    FlakeRef && flakeRef,
+    std::string_view fragment,
+    ExtendedOutputsSpec extendedOutputsSpec,
+    Strings attrPaths,
+    Strings prefixes,
+    const flake::LockFlags & lockFlags)
+    : InstallableValue(state),
+      flakeRef(flakeRef),
+      attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
+      prefixes(fragment == "" ? Strings{} : prefixes),
+      extendedOutputsSpec(std::move(extendedOutputsSpec)),
+      lockFlags(lockFlags)
+{
+    if (cmd && cmd->getAutoArgs(*state)->size())
+        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
+DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
+{
+    Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
+
+    auto attr = getCursor(*state);
+
+    auto attrPath = attr->getAttrPathStr();
+
+    if (!attr->isDerivation()) {
+
+        // FIXME: use eval cache?
+        auto v = attr->forceValue();
+
+        if (v.type() == nPath) {
+            PathSet context;
+            auto storePath = state->copyPathToStore(context, Path(v.path));
+            return {{
+                .path = DerivedPath::Opaque {
+                    .path = std::move(storePath),
+                }
+            }};
+        }
+
+        else if (v.type() == nString) {
+            PathSet context;
+            auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
+            auto storePath = state->store->maybeParseStorePath(s);
+            if (storePath && context.count(std::string(s))) {
+                return {{
+                    .path = DerivedPath::Opaque {
+                        .path = std::move(*storePath),
+                    }
+                }};
+            } else
+                throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
+        }
+
+        else
+            throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
+    }
+
+    auto drvPath = attr->forceDerivation();
+
+    std::optional<NixInt> priority;
+
+    if (attr->maybeGetAttr(state->sOutputSpecified)) {
+    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt();
+    }
+
+    return {{
+        .path = DerivedPath::Built {
+            .drvPath = std::move(drvPath),
+            .outputs = std::visit(overloaded {
+                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
+                    std::set<std::string> outputsToInstall;
+                    if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
+                        if (aOutputSpecified->getBool()) {
+                            if (auto aOutputName = attr->maybeGetAttr("outputName"))
+                                outputsToInstall = { aOutputName->getString() };
+                        }
+                    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
+                        if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
+                            for (auto & s : aOutputsToInstall->getListOfStrings())
+                                outputsToInstall.insert(s);
+                    }
+
+                    if (outputsToInstall.empty())
+                        outputsToInstall.insert("out");
+
+                    return OutputsSpec::Names { std::move(outputsToInstall) };
+                },
+                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
+                    return e;
+                },
+            }, extendedOutputsSpec.raw()),
+        },
+        .info = {
+            .priority = priority,
+            .originalRef = flakeRef,
+            .resolvedRef = getLockedFlake()->flake.lockedRef,
+            .attrPath = attrPath,
+            .extendedOutputsSpec = extendedOutputsSpec,
+        }
+    }};
+}
+
+std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
+{
+    return {&getCursor(state)->forceValue(), noPos};
+}
+
+std::vector<ref<eval_cache::AttrCursor>>
+InstallableFlake::getCursors(EvalState & state)
+{
+    auto evalCache = openEvalCache(state,
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
+
+    auto root = evalCache->getRoot();
+
+    std::vector<ref<eval_cache::AttrCursor>> res;
+
+    Suggestions suggestions;
+    auto attrPaths = getActualAttrPaths();
+
+    for (auto & attrPath : attrPaths) {
+        debug("trying flake output attribute '%s'", attrPath);
+
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+        if (attr) {
+            res.push_back(ref(*attr));
+        } else {
+            suggestions += attr.getSuggestions();
+        }
+    }
+
+    if (res.size() == 0)
+        throw Error(
+            suggestions,
+            "flake '%s' does not provide attribute %s",
+            flakeRef,
+            showAttrPaths(attrPaths));
+
+    return res;
+}
+
+std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
+{
+    if (!_lockedFlake) {
+        flake::LockFlags lockFlagsApplyConfig = lockFlags;
+        lockFlagsApplyConfig.applyNixConfig = true;
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
+    }
+    return _lockedFlake;
+}
+
+FlakeRef InstallableFlake::nixpkgsFlakeRef() const
+{
+    auto lockedFlake = getLockedFlake();
+
+    if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
+        if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
+            debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
+            return std::move(lockedNode->lockedRef);
+        }
+    }
+
+    return Installable::nixpkgsFlakeRef();
+}
+
+}

src/libcmd/installable-flake.hh

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "installable-value.hh"
+
+namespace nix {
+
+struct InstallableFlake : InstallableValue
+{
+    FlakeRef flakeRef;
+    Strings attrPaths;
+    Strings prefixes;
+    ExtendedOutputsSpec extendedOutputsSpec;
+    const flake::LockFlags & lockFlags;
+    mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
+
+    InstallableFlake(
+        SourceExprCommand * cmd,
+        ref<EvalState> state,
+        FlakeRef && flakeRef,
+        std::string_view fragment,
+        ExtendedOutputsSpec extendedOutputsSpec,
+        Strings attrPaths,
+        Strings prefixes,
+        const flake::LockFlags & lockFlags);
+
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+
+    std::vector<std::string> getActualAttrPaths();
+
+    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+
+    /* Get a cursor to every attrpath in getActualAttrPaths()
+       that exists. However if none exists, throw an exception. */
+    std::vector<ref<eval_cache::AttrCursor>>
+    getCursors(EvalState & state) override;
+
+    std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
+
+    FlakeRef nixpkgsFlakeRef() const override;
+};
+
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
+
+}

src/libcmd/installable-value.hh

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "installables.hh"
+
+namespace nix {
+
+struct InstallableValue : Installable
+{
+    ref<EvalState> state;
+
+    InstallableValue(ref<EvalState> state) : state(state) {}
+};
+
+}

src/libcmd/installables.cc

@@ -1,5 +1,8 @@
 #include "globals.hh"
 #include "installables.hh"
+#include "installable-derived-path.hh"
+#include "installable-attr-path.hh"
+#include "installable-flake.hh"
 #include "outputs-spec.hh"
 #include "util.hh"
 #include "command.hh"
@@ -144,7 +147,7 @@ void MixFlakeOptions::completionHook()
         completeFlakeInput(*prefix);
 }
 
-SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode)
+SourceExprCommand::SourceExprCommand()
 {
     addFlag({
         .longName = "file",
@@ -166,24 +169,18 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode)
         .labels = {"expr"},
         .handler = {&expr}
     });
+}
 
-    addFlag({
+MixReadOnlyOption::MixReadOnlyOption()
-        .longName = "derivation",
+{
-        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
-        .category = installablesCategory,
-        .handler = {&operateOn, OperateOn::Derivation},
-    });
-
-    if (supportReadOnlyMode) {
     addFlag({
         .longName = "read-only",
         .description =
             "Do not instantiate each evaluated derivation. "
             "This improves performance, but can cause errors when accessing "
             "store paths of derivations during evaluation.",
-            .handler = {&readOnlyMode, true},
+        .handler = {&settings.readOnlyMode, true},
     });
-    }
 }
 
 Strings SourceExprCommand::getDefaultFlakeAttrPaths()
@@ -396,143 +393,6 @@ static StorePath getDeriver(
     return *derivers.begin();
 }
 
-struct InstallableStorePath : Installable
-{
-    ref<Store> store;
-    DerivedPath req;
-
-    InstallableStorePath(ref<Store> store, DerivedPath && req)
-        : store(store), req(std::move(req))
-    { }
-
-    std::string what() const override
-    {
-        return req.to_string(*store);
-    }
-
-    DerivedPathsWithInfo toDerivedPaths() override
-    {
-        return {{.path = req, .info = {} }};
-    }
-
-    std::optional<StorePath> getStorePath() override
-    {
-        return std::visit(overloaded {
-            [&](const DerivedPath::Built & bfd) {
-                return bfd.drvPath;
-            },
-            [&](const DerivedPath::Opaque & bo) {
-                return bo.path;
-            },
-        }, req.raw());
-    }
-};
-
-struct InstallableAttrPath : InstallableValue
-{
-    SourceExprCommand & cmd;
-    RootValue v;
-    std::string attrPath;
-    ExtendedOutputsSpec extendedOutputsSpec;
-
-    InstallableAttrPath(
-        ref<EvalState> state,
-        SourceExprCommand & cmd,
-        Value * v,
-        const std::string & attrPath,
-        ExtendedOutputsSpec extendedOutputsSpec)
-        : InstallableValue(state)
-        , cmd(cmd)
-        , v(allocRootValue(v))
-        , attrPath(attrPath)
-        , extendedOutputsSpec(std::move(extendedOutputsSpec))
-    { }
-
-    std::string what() const override { return attrPath; }
-
-    std::pair<Value *, PosIdx> toValue(EvalState & state) override
-    {
-        auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
-        state.forceValue(*vRes, pos);
-        return {vRes, pos};
-    }
-
-    DerivedPathsWithInfo toDerivedPaths() override
-    {
-        auto v = toValue(*state).first;
-
-        Bindings & autoArgs = *cmd.getAutoArgs(*state);
-
-        DrvInfos drvInfos;
-        getDerivations(*state, *v, "", autoArgs, drvInfos, false);
-
-        // Backward compatibility hack: group results by drvPath. This
-        // helps keep .all output together.
-        std::map<StorePath, OutputsSpec> byDrvPath;
-
-        for (auto & drvInfo : drvInfos) {
-            auto drvPath = drvInfo.queryDrvPath();
-            if (!drvPath)
-                throw Error("'%s' is not a derivation", what());
-
-            auto newOutputs = std::visit(overloaded {
-                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
-                    std::set<std::string> outputsToInstall;
-                    for (auto & output : drvInfo.queryOutputs(false, true))
-                        outputsToInstall.insert(output.first);
-                    return OutputsSpec::Names { std::move(outputsToInstall) };
-                },
-                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
-                    return e;
-                },
-            }, extendedOutputsSpec.raw());
-
-            auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
-
-            if (!didInsert)
-                iter->second = iter->second.union_(newOutputs);
-        }
-
-        DerivedPathsWithInfo res;
-        for (auto & [drvPath, outputs] : byDrvPath)
-            res.push_back({
-                .path = DerivedPath::Built {
-                    .drvPath = drvPath,
-                    .outputs = outputs,
-                },
-            });
-
-        return res;
-    }
-};
-
-std::vector<std::string> InstallableFlake::getActualAttrPaths()
-{
-    std::vector<std::string> res;
-
-    for (auto & prefix : prefixes)
-        res.push_back(prefix + *attrPaths.begin());
-
-    for (auto & s : attrPaths)
-        res.push_back(s);
-
-    return res;
-}
-
-Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
-{
-    auto vFlake = state.allocValue();
-
-    callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
-    assert(aOutputs);
-
-    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
-
-    return aOutputs->value;
-}
-
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
@@ -562,196 +422,11 @@ ref<eval_cache::EvalCache> openEvalCache(
         });
 }
 
-static std::string showAttrPaths(const std::vector<std::string> & paths)
-{
-    std::string s;
-    for (const auto & [n, i] : enumerate(paths)) {
-        if (n > 0) s += n + 1 == paths.size() ? " or " : ", ";
-        s += '\''; s += i; s += '\'';
-    }
-    return s;
-}
-
-InstallableFlake::InstallableFlake(
-    SourceExprCommand * cmd,
-    ref<EvalState> state,
-    FlakeRef && flakeRef,
-    std::string_view fragment,
-    ExtendedOutputsSpec extendedOutputsSpec,
-    Strings attrPaths,
-    Strings prefixes,
-    const flake::LockFlags & lockFlags)
-    : InstallableValue(state),
-      flakeRef(flakeRef),
-      attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
-      prefixes(fragment == "" ? Strings{} : prefixes),
-      extendedOutputsSpec(std::move(extendedOutputsSpec)),
-      lockFlags(lockFlags)
-{
-    if (cmd && cmd->getAutoArgs(*state)->size())
-        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
-}
-
-DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
-{
-    Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
-
-    auto attr = getCursor(*state);
-
-    auto attrPath = attr->getAttrPathStr();
-
-    if (!attr->isDerivation()) {
-
-        // FIXME: use eval cache?
-        auto v = attr->forceValue();
-
-        if (v.type() == nPath) {
-            PathSet context;
-            auto storePath = state->copyPathToStore(context, Path(v.path));
-            return {{
-                .path = DerivedPath::Opaque {
-                    .path = std::move(storePath),
-                }
-            }};
-        }
-
-        else if (v.type() == nString) {
-            PathSet context;
-            auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
-            auto storePath = state->store->maybeParseStorePath(s);
-            if (storePath && context.count(std::string(s))) {
-                return {{
-                    .path = DerivedPath::Opaque {
-                        .path = std::move(*storePath),
-                    }
-                }};
-            } else
-                throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
-        }
-
-        else
-            throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
-    }
-
-    auto drvPath = attr->forceDerivation();
-
-    std::optional<NixInt> priority;
-
-    if (attr->maybeGetAttr(state->sOutputSpecified)) {
-    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
-        if (auto aPriority = aMeta->maybeGetAttr("priority"))
-            priority = aPriority->getInt();
-    }
-
-    return {{
-        .path = DerivedPath::Built {
-            .drvPath = std::move(drvPath),
-            .outputs = std::visit(overloaded {
-                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
-                    std::set<std::string> outputsToInstall;
-                    if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
-                        if (aOutputSpecified->getBool()) {
-                            if (auto aOutputName = attr->maybeGetAttr("outputName"))
-                                outputsToInstall = { aOutputName->getString() };
-                        }
-                    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
-                        if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
-                            for (auto & s : aOutputsToInstall->getListOfStrings())
-                                outputsToInstall.insert(s);
-                    }
-
-                    if (outputsToInstall.empty())
-                        outputsToInstall.insert("out");
-
-                    return OutputsSpec::Names { std::move(outputsToInstall) };
-                },
-                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
-                    return e;
-                },
-            }, extendedOutputsSpec.raw()),
-        },
-        .info = {
-            .priority = priority,
-            .originalRef = flakeRef,
-            .resolvedRef = getLockedFlake()->flake.lockedRef,
-            .attrPath = attrPath,
-            .extendedOutputsSpec = extendedOutputsSpec,
-        }
-    }};
-}
-
-std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
-{
-    return {&getCursor(state)->forceValue(), noPos};
-}
-
-std::vector<ref<eval_cache::AttrCursor>>
-InstallableFlake::getCursors(EvalState & state)
-{
-    auto evalCache = openEvalCache(state,
-        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
-
-    auto root = evalCache->getRoot();
-
-    std::vector<ref<eval_cache::AttrCursor>> res;
-
-    Suggestions suggestions;
-    auto attrPaths = getActualAttrPaths();
-
-    for (auto & attrPath : attrPaths) {
-        debug("trying flake output attribute '%s'", attrPath);
-
-        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
-        if (attr) {
-            res.push_back(ref(*attr));
-        } else {
-            suggestions += attr.getSuggestions();
-        }
-    }
-
-    if (res.size() == 0)
-        throw Error(
-            suggestions,
-            "flake '%s' does not provide attribute %s",
-            flakeRef,
-            showAttrPaths(attrPaths));
-
-    return res;
-}
-
-std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
-{
-    if (!_lockedFlake) {
-        flake::LockFlags lockFlagsApplyConfig = lockFlags;
-        lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
-    }
-    return _lockedFlake;
-}
-
-FlakeRef InstallableFlake::nixpkgsFlakeRef() const
-{
-    auto lockedFlake = getLockedFlake();
-
-    if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
-        if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
-            debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
-            return std::move(lockedNode->lockedRef);
-        }
-    }
-
-    return Installable::nixpkgsFlakeRef();
-}
-
 std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
     ref<Store> store, std::vector<std::string> ss)
 {
     std::vector<std::shared_ptr<Installable>> result;
 
-    if (readOnlyMode) {
-        settings.readOnlyMode = true;
-    }
-
     if (file || expr) {
         if (file && expr)
             throw UsageError("'--file' and '--expr' are exclusive");
@@ -777,9 +452,8 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s);
             result.push_back(
                 std::make_shared<InstallableAttrPath>(
-                    state, *this, vFile,
+                    InstallableAttrPath::parse(
-                    prefix == "." ? "" : std::string { prefix },
+                        state, *this, vFile, prefix, extendedOutputsSpec)));
-                    extendedOutputsSpec));
         }
 
     } else {
@@ -792,41 +466,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             auto prefix = std::move(prefix_);
             auto extendedOutputsSpec = std::move(extendedOutputsSpec_);
 
-            auto found = prefix.find('/');
+            if (prefix.find('/') != std::string::npos) {
-            if (found != std::string::npos) {
                 try {
-                    auto derivedPath = std::visit(overloaded {
+                    result.push_back(std::make_shared<InstallableDerivedPath>(
-                        // If the user did not use ^, we treat the output more liberally.
+                        InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec)));
-                        [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
-                            // First, we accept a symlink chain or an actual store path.
-                            auto storePath = store->followLinksToStorePath(prefix);
-                            // Second, we see if the store path ends in `.drv` to decide what sort
-                            // of derived path they want.
-                            //
-                            // This handling predates the `^` syntax. The `^*` in
-                            // `/nix/store/hash-foo.drv^*` unambiguously means "do the
-                            // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could
-                            // also unambiguously mean "do the DerivedPath::Opaque` case".
-                            //
-                            // Issue #7261 tracks reconsidering this `.drv` dispatching.
-                            return storePath.isDerivation()
-                                ? (DerivedPath) DerivedPath::Built {
-                                    .drvPath = std::move(storePath),
-                                    .outputs = OutputsSpec::All {},
-                                }
-                                : (DerivedPath) DerivedPath::Opaque {
-                                    .path = std::move(storePath),
-                                };
-                        },
-                        // If the user did use ^, we just do exactly what is written.
-                        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
-                            return DerivedPath::Built {
-                                .drvPath = store->parseStorePath(prefix),
-                                .outputs = outputSpec,
-                            };
-                        },
-                    }, extendedOutputsSpec.raw());
-                    result.push_back(std::make_shared<InstallableStorePath>(store, std::move(derivedPath)));
                     continue;
                 } catch (BadStorePath &) {
                 } catch (...) {
@@ -1062,8 +705,8 @@ void InstallablesCommand::prepare()
     installables = load();
 }
 
-Installables InstallablesCommand::load() {
+Installables InstallablesCommand::load()
-    Installables installables;
+{
     if (_installables.empty() && useDefaultInstallables())
         // FIXME: commands like "nix profile install" should not have a
         // default, probably.
@@ -1073,16 +716,13 @@ Installables InstallablesCommand::load() {
 
 std::vector<std::string> InstallablesCommand::getFlakesForCompletion()
 {
-    if (_installables.empty()) {
+    if (_installables.empty() && useDefaultInstallables())
-        if (useDefaultInstallables())
         return {"."};
-        return {};
-    }
     return _installables;
 }
 
-InstallableCommand::InstallableCommand(bool supportReadOnlyMode)
+InstallableCommand::InstallableCommand()
-    : SourceExprCommand(supportReadOnlyMode)
+    : SourceExprCommand()
 {
     expectArgs({
         .label = "installable",

src/libcmd/installables.hh

@@ -161,54 +161,4 @@ struct Installable
 
 typedef std::vector<std::shared_ptr<Installable>> Installables;
 
-struct InstallableValue : Installable
-{
-    ref<EvalState> state;
-
-    InstallableValue(ref<EvalState> state) : state(state) {}
-};
-
-struct InstallableFlake : InstallableValue
-{
-    FlakeRef flakeRef;
-    Strings attrPaths;
-    Strings prefixes;
-    ExtendedOutputsSpec extendedOutputsSpec;
-    const flake::LockFlags & lockFlags;
-    mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
-
-    InstallableFlake(
-        SourceExprCommand * cmd,
-        ref<EvalState> state,
-        FlakeRef && flakeRef,
-        std::string_view fragment,
-        ExtendedOutputsSpec extendedOutputsSpec,
-        Strings attrPaths,
-        Strings prefixes,
-        const flake::LockFlags & lockFlags);
-
-    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
-
-    std::vector<std::string> getActualAttrPaths();
-
-    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
-
-    DerivedPathsWithInfo toDerivedPaths() override;
-
-    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
-
-    /* Get a cursor to every attrpath in getActualAttrPaths()
-       that exists. However if none exists, throw an exception. */
-    std::vector<ref<eval_cache::AttrCursor>>
-    getCursors(EvalState & state) override;
-
-    std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
-
-    FlakeRef nixpkgsFlakeRef() const override;
-};
-
-ref<eval_cache::EvalCache> openEvalCache(
-    EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake);
-
 }

src/libcmd/local.mk

@@ -6,7 +6,7 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
+libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 

src/libcmd/nix-cmd.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixcmd
-Cflags: -I${includedir}/nix -std=c++20
+Cflags: -I${includedir}/nix -std=c++2a

src/libcmd/repl.cc

@@ -19,6 +19,8 @@ extern "C" {
 }
 #endif
 
+#include "repl.hh"
+
 #include "ansicolor.hh"
 #include "shared.hh"
 #include "eval.hh"
@@ -31,7 +33,9 @@ extern "C" {
 #include "get-drvs.hh"
 #include "derivations.hh"
 #include "globals.hh"
-#include "command.hh"
+#include "flake/flake.hh"
+#include "flake/lockfile.hh"
+#include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
@@ -45,18 +49,16 @@ extern "C" {
 namespace nix {
 
 struct NixRepl
+    : AbstractNixRepl
     #if HAVE_BOEHMGC
-    : gc
+    , gc
     #endif
 {
     std::string curDir;
-    ref<EvalState> state;
-    Bindings * autoArgs;
 
     size_t debugTraceIndex;
 
     Strings loadedFiles;
-    typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
     std::function<AnnotatedValues()> getValues;
 
     const static int envSize = 32768;
@@ -69,8 +71,11 @@ struct NixRepl
 
     NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state,
             std::function<AnnotatedValues()> getValues);
-    ~NixRepl();
+    virtual ~NixRepl();
-    void mainLoop();
+
+    void mainLoop() override;
+    void initEnv() override;
+
     StringSet completePrefix(const std::string & prefix);
     bool getLine(std::string & input, const std::string & prompt);
     StorePath getDerivationPath(Value & v);
@@ -78,7 +83,6 @@ struct NixRepl
 
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
-    void initEnv();
     void loadFiles();
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
@@ -92,7 +96,6 @@ struct NixRepl
     std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
 };
 
-
 std::string removeWhitespace(std::string s)
 {
     s = chomp(s);
@@ -104,7 +107,7 @@ std::string removeWhitespace(std::string s)
 
 NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
             std::function<NixRepl::AnnotatedValues()> getValues)
-    : state(state)
+    : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
@@ -1029,8 +1032,22 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     return str;
 }
 
-void runRepl(
+
-    ref<EvalState>evalState,
+std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
+   const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+   std::function<AnnotatedValues()> getValues)
+{
+    return std::make_unique<NixRepl>(
+        searchPath,
+        openStore(),
+        state,
+        getValues
+    );
+}
+
+
+void AbstractNixRepl::runSimple(
+    ref<EvalState> evalState,
     const ValMap & extraEnv)
 {
     auto getValues = [&]()->NixRepl::AnnotatedValues{
@@ -1054,91 +1071,4 @@ void runRepl(
     repl->mainLoop();
 }
 
-struct CmdRepl : InstallablesCommand
-{
-    CmdRepl() {
-        evalSettings.pureEval = false;
-    }
-
-    void prepare() override
-    {
-        if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) {
-            warn("future versions of Nix will require using `--file` to load a file");
-            if (this->_installables.size() > 1)
-                warn("more than one input file is not currently supported");
-            auto filePath = this->_installables[0].data();
-            file = std::optional(filePath);
-            _installables.front() = _installables.back();
-            _installables.pop_back();
-        }
-        installables = InstallablesCommand::load();
-    }
-
-    std::vector<std::string> files;
-
-    Strings getDefaultFlakeAttrPaths() override
-    {
-        return {""};
-    }
-
-    bool useDefaultInstallables() override
-    {
-        return file.has_value() or expr.has_value();
-    }
-
-    bool forceImpureByDefault() override
-    {
-        return true;
-    }
-
-    std::string description() override
-    {
-        return "start an interactive environment for evaluating Nix expressions";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "repl.md"
-          ;
-    }
-
-    void run(ref<Store> store) override
-    {
-        auto state = getEvalState();
-        auto getValues = [&]()->NixRepl::AnnotatedValues{
-            auto installables = load();
-            NixRepl::AnnotatedValues values;
-            for (auto & installable: installables){
-                auto what = installable->what();
-                if (file){
-                    auto [val, pos] = installable->toValue(*state);
-                    auto what = installable->what();
-                    state->forceValue(*val, pos);
-                    auto autoArgs = getAutoArgs(*state);
-                    auto valPost = state->allocValue();
-                    state->autoCallFunction(*autoArgs, *val, *valPost);
-                    state->forceValue(*valPost, pos);
-                    values.push_back( {valPost, what });
-                } else {
-                    auto [val, pos] = installable->toValue(*state);
-                    values.push_back( {val, what} );
-                }
-            }
-            return values;
-        };
-        auto repl = std::make_unique<NixRepl>(
-            searchPath,
-            openStore(),
-            state,
-            getValues
-        );
-        repl->autoArgs = getAutoArgs(*repl->state);
-        repl->initEnv();
-        repl->mainLoop();
-    }
-};
-
-static auto rCmdRepl = registerCommand<CmdRepl>("repl");
-
 }

src/libcmd/repl.hh

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "eval.hh"
+
+#if HAVE_BOEHMGC
+#define GC_INCLUDE_NEW
+#include <gc/gc_cpp.h>
+#endif
+
+namespace nix {
+
+struct AbstractNixRepl
+{
+    ref<EvalState> state;
+    Bindings * autoArgs;
+
+    AbstractNixRepl(ref<EvalState> state)
+        : state(state)
+    { }
+
+    virtual ~AbstractNixRepl()
+    { }
+
+    typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
+
+    static std::unique_ptr<AbstractNixRepl> create(
+        const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+        std::function<AnnotatedValues()> getValues);
+
+    static void runSimple(
+        ref<EvalState> evalState,
+        const ValMap & extraEnv);
+
+    virtual void initEnv() = 0;
+
+    virtual void mainLoop() = 0;
+};
+
+}

src/libexpr/eval.cc

@@ -519,7 +519,6 @@ EvalState::EvalState(
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
     /* Initialise the Nix expression search path. */
-    evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath());
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
@@ -2473,35 +2472,30 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 
 EvalSettings::EvalSettings()
 {
+    auto var = getEnv("NIX_PATH");
+    if (var) nixPath = parseNixPath(*var);
 }
 
-/* impure => NIX_PATH or a default path
- * restrict-eval => NIX_PATH
- * pure-eval => empty
- */
 Strings EvalSettings::getDefaultNixPath()
 {
-    if (pureEval)
-        return {};
-
-    auto var = getEnv("NIX_PATH");
-    if (var) {
-        return parseNixPath(*var);
-    } else if (restrictEval) {
-        return {};
-    } else {
     Strings res;
-        auto add = [&](const Path & p, const std::optional<std::string> & s = std::nullopt) {
+    auto add = [&](const Path & p, const std::string & s = std::string()) {
-            if (pathExists(p))
+        if (pathExists(p)) {
-                res.push_back(s ? *s + "=" + p : p);
+            if (s.empty()) {
+                res.push_back(p);
+            } else {
+                res.push_back(s + "=" + p);
+            }
+        }
     };
 
-        add(getHome() + "/.nix-defexpr/channels");
+    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
+        add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
         add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
         add(settings.nixStateDir + "/profiles/per-user/root/channels");
+    }
 
     return res;
-    }
 }
 
 bool EvalSettings::isPseudoUrl(std::string_view s)

src/libexpr/eval.hh

@@ -570,7 +570,7 @@ struct EvalSettings : Config
 {
     EvalSettings();
 
-    Strings getDefaultNixPath();
+    static Strings getDefaultNixPath();
 
     static bool isPseudoUrl(std::string_view s);
 
@@ -580,15 +580,8 @@ struct EvalSettings : Config
         "Whether builtin functions that allow executing native code should be enabled."};
 
     Setting<Strings> nixPath{
-        this, {}, "nix-path",
+        this, getDefaultNixPath(), "nix-path",
-        R"(
+        "List of directories to be searched for `<...>` file references."};
-          List of directories to be searched for `<...>` file references.
-
-          If [pure evaluation](#conf-pure-eval) is disabled,
-          this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH)
-          environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval)
-          is disabled, a default search path including the user's and `root`'s channels.
-        )"};
 
     Setting<bool> restrictEval{
         this, false, "restrict-eval",

src/libexpr/flake/call-flake.nix

@@ -16,7 +16,9 @@ let
 
           subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
 
-          flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
+          outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
+
+          flake = import (outPath + "/flake.nix");
 
           inputs = builtins.mapAttrs
             (inputName: inputSpec: allNodes.${resolveInput inputSpec})
@@ -43,7 +45,21 @@ let
 
           outputs = flake.outputs (inputs // { self = result; });
 
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; };
+          result =
+            outputs
+            # We add the sourceInfo attribute for its metadata, as they are
+            # relevant metadata for the flake. However, the outPath of the
+            # sourceInfo does not necessarily match the outPath of the flake,
+            # as the flake may be in a subdirectory of a source.
+            # This is shadowed in the next //
+            // sourceInfo
+            // {
+              # This shadows the sourceInfo.outPath
+              inherit outPath;
+
+              inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake";
+            };
+
         in
           if node.flake or true then
             assert builtins.isFunction flake.outputs;

src/libexpr/nix-expr.pc.in

@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++20
+Cflags: -I${includedir}/nix -std=c++2a

src/libexpr/nixexpr.hh

@@ -186,7 +186,7 @@ struct ExprString : Expr
 {
     std::string s;
     Value v;
-    ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); };
+    ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); };
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };
@@ -233,7 +233,7 @@ struct ExprSelect : Expr
     PosIdx pos;
     Expr * e, * def;
     AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
+    ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -243,7 +243,7 @@ struct ExprOpHasAttr : Expr
 {
     Expr * e;
     AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { };
+    ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -90,7 +90,7 @@ static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, cons
 }
 
 
-static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
+static void addAttr(ExprAttrs * attrs, AttrPath && attrPath,
     Expr * e, const PosIdx pos, const nix::EvalState & state)
 {
     AttrPath::iterator i;
@@ -188,7 +188,7 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals,
 
 
 static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols,
-    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> & es)
+    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
 {
     if (es.empty()) return new ExprString("");
 
@@ -268,7 +268,7 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols,
                 s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->emplace_back(i->first, new ExprString(s2));
+        es2->emplace_back(i->first, new ExprString(std::move(s2)));
     };
     for (; i != es.end(); ++i, --n) {
         std::visit(overloaded { trimExpr, trimString }, i->second);
@@ -408,7 +408,7 @@ expr_op
   | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); }
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
+  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
   | expr_op '+' expr_op
     { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector<std::pair<PosIdx, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
@@ -431,14 +431,14 @@ expr_app
 
 expr_select
   : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); }
+    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
   | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); }
+    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; }
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
     { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
-  | expr_simple { $$ = $1; }
+  | expr_simple
   ;
 
 expr_simple
@@ -453,9 +453,10 @@ expr_simple
   | FLOAT { $$ = new ExprFloat($1); }
   | '"' string_parts '"' { $$ = $2; }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = stripIndentation(CUR_POS, data->symbols, *$2);
+      $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2));
+      delete $2;
   }
-  | path_start PATH_END { $$ = $1; }
+  | path_start PATH_END
   | path_start string_parts_interpolated PATH_END {
       $2->insert($2->begin(), {makeCurPos(@1, data), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
@@ -465,7 +466,7 @@ expr_simple
       $$ = new ExprCall(CUR_POS,
           new ExprVar(data->symbols.create("__findFile")),
           {new ExprVar(data->symbols.create("__nixPath")),
-           new ExprString(path)});
+           new ExprString(std::move(path))});
   }
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
@@ -533,7 +534,7 @@ ind_string_parts
   ;
 
 binds
-  : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data), data->state); }
+  : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; }
   | binds INHERIT attrs ';'
     { $$ = $1;
       for (auto & i : *$3) {
@@ -542,6 +543,7 @@ binds
           auto pos = makeCurPos(@3, data);
           $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
       }
+      delete $3;
     }
   | binds INHERIT '(' expr ')' attrs ';'
     { $$ = $1;
@@ -551,6 +553,7 @@ binds
               dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
           $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
       }
+      delete $6;
     }
   | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
   ;
@@ -596,7 +599,7 @@ attrpath
   ;
 
 attr
-  : ID { $$ = $1; }
+  : ID
   | OR_KW { $$ = {"or", 2}; }
   ;
 
@@ -612,9 +615,9 @@ expr_list
 
 formals
   : formal ',' formals
-    { $$ = $3; $$->formals.push_back(*$1); }
+    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
   | formal
-    { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; }
+    { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
   |
     { $$ = new ParserFormals; $$->ellipsis = false; }
   | ELLIPSIS

src/libexpr/primops.cc

@@ -3044,9 +3044,9 @@ static RegisterPrimOp primop_foldlStrict({
     .doc = R"(
       Reduce a list by applying a binary operator, from left to right,
       e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
-      ...`. The operator is applied strictly, i.e., its arguments are
+      ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6.
-      evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]`
+      The return value of each application of `op` is evaluated immediately,
-      evaluates to 6.
+      even for intermediate values.
     )",
     .fun = prim_foldlStrict,
 });

src/libexpr/primops/fetchTree.cc

@@ -4,6 +4,7 @@
 #include "fetchers.hh"
 #include "filetransfer.hh"
 #include "registry.hh"
+#include "url.hh"
 
 #include <ctime>
 #include <iomanip>
@@ -68,7 +69,16 @@ void emitTreeAttrs(
 std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
-    return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
+    if (uri.find("://") == std::string::npos) {
+        const auto p = ParsedURL {
+            .scheme = defaultScheme,
+            .authority = "",
+            .path = uri
+        };
+        return p.to_string();
+    } else {
+        return uri;
+    }
 }
 
 std::string fixURIForGit(std::string uri, EvalState & state)
@@ -461,6 +471,17 @@ static RegisterPrimOp primop_fetchGit({
           > **Note**
           >
           > This behavior is disabled in *Pure evaluation mode*.
+
+        - To fetch the content of a checked-out work directory:
+
+          ```nix
+          builtins.fetchGit ./work-dir
+          ```
+
+	  If the URL points to a local directory, and no `ref` or `rev` is
+	  given, `fetchGit` will use the current content of the checked-out
+	  files, even if they are not committed or added to Git's index. It will
+	  only consider files added to the Git repository, as listed by `git ls-files`.
     )",
     .fun = prim_fetchGit,
 });

src/libfetchers/fetch-settings.hh

@@ -57,7 +57,7 @@ struct FetchSettings : public Config
           ```
 
           This example specifies three tokens, one each for accessing
-          github.com, gitlab.mycompany.com, and sourceforge.net.
+          github.com, gitlab.mycompany.com, and gitlab.com.
 
           The `input.foo` uses the "gitlab" fetcher, which might
           requires specifying the token type along with the token

src/libfetchers/git.cc

@@ -615,15 +615,42 @@ struct GitInputScheme : InputScheme
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
             runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
+
+            {
                 // TODO: repoDir might lack the ref (it only checks if rev
                 // exists, see FIXME above) so use a big hammer and fetch
                 // everything to ensure we get the rev.
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
                 runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+            }
 
             runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
-            runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
+
+            /* Ensure that we use the correct origin for fetching
+               submodules. This matters for submodules with relative
+               URLs. */
+            if (isLocal) {
+                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config"));
+
+                /* Restore the config.bare setting we may have just
+                   copied erroneously from the user's repo. */
+                runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
+            } else
+                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl });
+
+            /* As an optimisation, copy the modules directory of the
+               source repo if it exists. */
+            auto modulesPath = repoDir + "/" + gitDir + "/modules";
+            if (pathExists(modulesPath)) {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl));
+                runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
+            }
+
+            {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
                 runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+            }
 
             filter = isNotDotGitDirectory;
         } else {

src/libmain/nix-main.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix -std=c++20
+Cflags: -I${includedir}/nix -std=c++2a

src/libstore/build/local-derivation-goal.cc

@@ -16,6 +16,7 @@
 #include "json-utils.hh"
 #include "cgroup.hh"
 #include "personality.hh"
+#include "namespaces.hh"
 
 #include <regex>
 #include <queue>
@@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats)
 }
 
 
-void LocalDerivationGoal::tryLocalBuild() {
+void LocalDerivationGoal::tryLocalBuild()
+{
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
         state = &DerivationGoal::tryToBuild;
@@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() {
         #endif
     }
 
+    #if __linux__
+    if (useChroot) {
+        if (!mountAndPidNamespacesSupported()) {
+            if (!settings.sandboxFallback)
+                throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing");
+            debug("auto-disabling sandboxing because the prerequisite namespaces are not available");
+            useChroot = false;
+        }
+    }
+    #endif
+
     if (useBuildUsers()) {
         if (!buildUser)
             buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
@@ -372,12 +385,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 }
 
 
-int childEntry(void * arg)
-{
-    ((LocalDerivationGoal *) arg)->runChild();
-    return 1;
-}
-
 #if __linux__
 static void linkOrCopy(const Path & from, const Path & to)
 {
@@ -663,6 +670,7 @@ void LocalDerivationGoal::startBuilder()
            nobody account.  The latter is kind of a hack to support
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
+        if (parsedDrv->useUidRange())
             chownToBuilder(chrootRootDir + "/etc");
 
         if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
@@ -888,12 +896,7 @@ void LocalDerivationGoal::startBuilder()
 
         userNamespaceSync.create();
 
-        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+        usingUserNamespace = userNamespacesSupported();
-        static bool userNamespacesEnabled =
-            pathExists(maxUserNamespaces)
-            && trim(readFile(maxUserNamespaces)) != "0";
-
-        usingUserNamespace = userNamespacesEnabled;
 
         Pid helper = startProcess([&]() {
 
@@ -908,76 +911,21 @@ void LocalDerivationGoal::startBuilder()
             if (getuid() == 0 && setgroups(0, 0) == -1)
                 throw SysError("setgroups failed");
 
-            size_t stackSize = 1 * 1024 * 1024;
+            ProcessOptions options;
-            char * stack = (char *) mmap(0, stackSize,
+            options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-                PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-            if (stack == MAP_FAILED) throw SysError("allocating stack");
-
-            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
             if (privateNetwork)
-                flags |= CLONE_NEWNET;
+                options.cloneFlags |= CLONE_NEWNET;
             if (usingUserNamespace)
-                flags |= CLONE_NEWUSER;
+                options.cloneFlags |= CLONE_NEWUSER;
+
+            pid_t child = startProcess([&]() { runChild(); }, options);
 
-            pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL) {
-                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
-                   CLONE_PARENT are not allowed together. */
-                flags &= ~CLONE_NEWPID;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unprivileged
-                 * user namespaces. If we get EPERM or EINVAL, try
-                 * without CLONE_NEWUSER and see if that works.
-                 * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6
-                 */
-                usingUserNamespace = false;
-                flags &= ~CLONE_NEWUSER;
-                child = clone(childEntry, stack + stackSize, flags, this);
-            }
-            if (child == -1) {
-                switch(errno) {
-                    case EPERM:
-                    case EINVAL: {
-                        int errno_ = errno;
-                        if (!userNamespacesEnabled && errno==EPERM)
-                            notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces");
-                        if (userNamespacesEnabled) {
-                            Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone";
-                            if (pathExists(procSysKernelUnprivilegedUsernsClone)
-                                && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") {
-                                notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone");
-                            }
-                        }
-                        Path procSelfNsUser = "/proc/self/ns/user";
-                        if (!pathExists(procSelfNsUser))
-                            notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing");
-                        /* Otherwise exit with EPERM so we can handle this in the
-                           parent. This is only done when sandbox-fallback is set
-                           to true (the default). */
-                        if (settings.sandboxFallback)
-                            _exit(1);
-                        /* Mention sandbox-fallback in the error message so the user
-                           knows that having it disabled contributed to the
-                           unrecoverability of this failure */
-                        throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback");
-                    }
-                    default:
-                        throw SysError("creating sandboxed builder process using clone()");
-                }
-            }
             writeFull(builderOut.writeSide.get(),
                 fmt("%d %d\n", usingUserNamespace, child));
             _exit(0);
         });
 
-        int res = helper.wait();
+        if (helper.wait() != 0)
-        if (res != 0 && settings.sandboxFallback) {
-            useChroot = false;
-            initTmpDir();
-            goto fallback;
-        } else if (res != 0)
             throw Error("unable to start build process");
 
         userNamespaceSync.readSide = -1;
@@ -1045,9 +993,6 @@ void LocalDerivationGoal::startBuilder()
     } else
 #endif
     {
-#if __linux__
-    fallback:
-#endif
         pid = startProcess([&]() {
             runChild();
         });
@@ -1516,8 +1461,7 @@ void LocalDerivationGoal::startDaemon()
                 FdSink to(remote.get());
                 try {
                     daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive,
+                        daemon::NotTrusted, daemon::Recursive);
-                        [&](Store & store) {});
                     debug("terminated daemon connection");
                 } catch (SysError &) {
                     ignoreException();
@@ -1907,6 +1851,10 @@ void LocalDerivationGoal::runChild()
                 }
             }
 
+            /* Make /etc unwritable */
+            if (!parsedDrv->useUidRange())
+                chmod_(chrootRootDir + "/etc", 0555);
+
             /* Unshare this mount namespace. This is necessary because
                pivot_root() below changes the root of the mount
                namespace. This means that the call to setns() in

src/libstore/daemon.cc

@@ -236,6 +236,10 @@ struct ClientSettings
                     // the daemon, as that could cause some pretty weird stuff
                     if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
                         debug("Ignoring the client-specified experimental features");
+                } else if (name == settings.pluginFiles.name) {
+                    if (tokenizeString<Paths>(value) != settings.pluginFiles.get())
+                        warn("Ignoring the client-specified plugin-files.\n"
+                             "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14.");
                 }
                 else if (trusted
                     || name == settings.buildTimeout.name
@@ -525,7 +529,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             mode = (BuildMode) readInt(from);
 
             /* Repairing is not atomic, so disallowed for "untrusted"
-               clients.  */
+               clients.
+
+               FIXME: layer violation in this message: the daemon code (i.e.
+               this file) knows whether a client/connection is trusted, but it
+               does not how how the client was authenticated. The mechanism
+               need not be getting the UID of the other end of a Unix Domain
+               Socket.
+              */
             if (mode == bmRepair && !trusted)
                 throw Error("repairing is not allowed because you are not in 'trusted-users'");
         }
@@ -542,7 +553,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         mode = (BuildMode) readInt(from);
 
         /* Repairing is not atomic, so disallowed for "untrusted"
-           clients.  */
+           clients.
+
+           FIXME: layer violation; see above. */
         if (mode == bmRepair && !trusted)
             throw Error("repairing is not allowed because you are not in 'trusted-users'");
 
@@ -981,8 +994,7 @@ void processConnection(
     FdSource & from,
     FdSink & to,
     TrustedFlag trusted,
-    RecursiveFlag recursive,
+    RecursiveFlag recursive)
-    std::function<void(Store &)> authHook)
 {
     auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
 
@@ -1025,10 +1037,6 @@ void processConnection(
 
     try {
 
-        /* If we can't accept clientVersion, then throw an error
-           *here* (not above). */
-        authHook(*store);
-
         tunnelLogger->stopWork();
         to.flush();
 

src/libstore/daemon.hh

@@ -13,11 +13,6 @@ void processConnection(
     FdSource & from,
     FdSink & to,
     TrustedFlag trusted,
-    RecursiveFlag recursive,
+    RecursiveFlag recursive);
-    /* Arbitrary hook to check authorization / initialize user data / whatever
-       after the protocol has been negotiated. The idea is that this function
-       and everything it calls doesn't know about this stuff, and the
-       `nix-daemon` handles that instead. */
-    std::function<void(Store &)> authHook);
 
 }

src/libstore/derivations.cc

@@ -5,6 +5,7 @@
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -890,4 +891,63 @@ std::optional<BasicDerivation> Derivation::tryResolve(
 
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
+nlohmann::json DerivationOutput::toJSON(
+    const Store & store, std::string_view drvName, std::string_view outputName) const
+{
+    nlohmann::json res = nlohmann::json::object();
+    std::visit(overloaded {
+        [&](const DerivationOutput::InputAddressed & doi) {
+            res["path"] = store.printStorePath(doi.path);
+        },
+        [&](const DerivationOutput::CAFixed & dof) {
+            res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
+            res["hashAlgo"] = dof.hash.printMethodAlgo();
+            res["hash"] = dof.hash.hash.to_string(Base16, false);
+        },
+        [&](const DerivationOutput::CAFloating & dof) {
+            res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
+        },
+        [&](const DerivationOutput::Deferred &) {},
+        [&](const DerivationOutput::Impure & doi) {
+            res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
+            res["impure"] = true;
+        },
+    }, raw());
+    return res;
+}
+
+nlohmann::json Derivation::toJSON(const Store & store) const
+{
+    nlohmann::json res = nlohmann::json::object();
+
+    {
+        nlohmann::json & outputsObj = res["outputs"];
+        outputsObj = nlohmann::json::object();
+        for (auto & [outputName, output] : outputs) {
+            outputsObj[outputName] = output.toJSON(store, name, outputName);
+        }
+    }
+
+    {
+        auto& inputsList = res["inputSrcs"];
+        inputsList = nlohmann::json ::array();
+        for (auto & input : inputSrcs)
+            inputsList.emplace_back(store.printStorePath(input));
+    }
+
+    {
+        auto& inputDrvsObj = res["inputDrvs"];
+        inputDrvsObj = nlohmann::json ::object();
+        for (auto & input : inputDrvs)
+            inputDrvsObj[store.printStorePath(input.first)] = input.second;
+    }
+
+    res["system"] = platform;
+    res["builder"] = builder;
+    res["args"] = args;
+    res["env"] = env;
+
+    return res;
+}
+
 }

src/libstore/derivations.hh

@@ -83,6 +83,11 @@ struct DerivationOutput : _DerivationOutputRaw
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
+
+    nlohmann::json toJSON(
+        const Store & store,
+        std::string_view drvName,
+        std::string_view outputName) const;
 };
 
 typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -210,6 +215,8 @@ struct Derivation : BasicDerivation
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
+
+    nlohmann::json toJSON(const Store & store) const;
 };
 
 

src/libstore/filetransfer.cc

@@ -101,6 +101,7 @@ struct curlFileTransfer : public FileTransfer
                     this->result.data.append(data);
               })
         {
+            requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
             if (!request.mimeType.empty())
@@ -828,7 +829,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
         {
             auto state(_state->lock());
 
-            while (state->data.empty()) {
+            if (state->data.empty()) {
 
                 if (state->quit) {
                     if (state->exc) std::rethrow_exception(state->exc);
@@ -836,6 +837,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
                 }
 
                 state.wait(state->avail);
+
+                if (state->data.empty()) continue;
             }
 
             chunk = std::move(state->data);

src/libstore/globals.hh

@@ -201,7 +201,16 @@ public:
         {"build-timeout"}};
 
     PathSetting buildHook{this, true, "", "build-hook",
-        "The path of the helper program that executes builds to remote machines."};
+        R"(
+          The path to the helper program that executes remote builds.
+
+          Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon.
+          The default value is the internal Nix binary that implements remote building.
+
+          > **Important**
+          >
+          > Change this setting only if you really know what you’re doing.
+        )"};
 
     Setting<std::string> builders{
         this, "@" + nixConfDir + "/machines", "builders",
@@ -279,8 +288,8 @@ public:
           If the build users group is empty, builds will be performed under
           the uid of the Nix process (that is, the uid of the caller if
           `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
-          `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
+          `NIX_REMOTE` is `daemon`). Obviously, this should not be used
-          multi-user settings with untrusted users.
+          with a nix daemon accessible to untrusted clients.
 
           Defaults to `nixbld` when running as root, *empty* otherwise.
         )",
@@ -696,24 +705,6 @@ public:
         )",
         {"trusted-binary-caches"}};
 
-    Setting<Strings> trustedUsers{
-        this, {"root"}, "trusted-users",
-        R"(
-          A list of names of users (separated by whitespace) that have
-          additional rights when connecting to the Nix daemon, such as the
-          ability to specify additional binary caches, or to import unsigned
-          NARs. You can also specify groups by prefixing them with `@`; for
-          instance, `@wheel` means all users in the `wheel` group. The default
-          is `root`.
-
-          > **Warning**
-          >
-          > Adding a user to `trusted-users` is essentially equivalent to
-          > giving that user root access to the system. For example, the user
-          > can set `sandbox-paths` and thereby obtain read access to
-          > directories that are otherwise inacessible to them.
-        )"};
-
     Setting<unsigned int> ttlNegativeNarInfoCache{
         this, 3600, "narinfo-cache-negative-ttl",
         R"(
@@ -736,18 +727,6 @@ public:
           mismatch if the build isn't reproducible.
         )"};
 
-    /* ?Who we trust to use the daemon in safe ways */
-    Setting<Strings> allowedUsers{
-        this, {"*"}, "allowed-users",
-        R"(
-          A list of names of users (separated by whitespace) that are allowed
-          to connect to the Nix daemon. As with the `trusted-users` option,
-          you can specify groups by prefixing them with `@`. Also, you can
-          allow all users by specifying `*`. The default is `*`.
-
-          Note that trusted users are always allowed to connect.
-        )"};
-
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
@@ -975,6 +954,27 @@ public:
           resolves to a different location from that of the build machine. You
           can enable this setting if you are sure you're not going to do that.
         )"};
+
+    Setting<bool> useXDGBaseDirectories{
+        this, false, "use-xdg-base-directories",
+        R"(
+          If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`.
+          The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md).
+
+          [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+          > **Warning**
+          > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations.
+
+          In particular, the following locations change:
+
+          | Old               | New                            |
+          |-------------------|--------------------------------|
+          | `~/.nix-profile`  | `$XDG_STATE_HOME/nix/profile`  |
+          | `~/.nix-defexpr`  | `$XDG_STATE_HOME/nix/defexpr`  |
+          | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
+        )"
+    };
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -56,7 +56,7 @@ public:
     void init() override
     {
         // FIXME: do this lazily?
-        if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
+        if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
             wantMassQuery.setDefault(cacheInfo->wantMassQuery);
             priority.setDefault(cacheInfo->priority);
         } else {

src/libstore/nar-info-disk-cache.cc

@@ -84,11 +84,10 @@ public:
 
     Sync<State> _state;
 
-    NarInfoDiskCacheImpl()
+    NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite")
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
@@ -98,7 +97,7 @@ public:
         state->db.exec(schema);
 
         state->insertCache.create(state->db,
-            "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+            "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;");
 
         state->queryCache.create(state->db,
             "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
@@ -166,6 +165,8 @@ public:
         return i->second;
     }
 
+private:
+
     std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
     {
         auto i = state.caches.find(uri);
@@ -173,15 +174,21 @@ public:
             auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
             if (!queryCache.next())
                 return std::nullopt;
-            state.caches.emplace(uri,
+            auto cache = Cache {
-                Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+                .id = (int) queryCache.getInt(0),
+                .storeDir = queryCache.getStr(1),
+                .wantMassQuery = queryCache.getInt(2) != 0,
+                .priority = (int) queryCache.getInt(3),
+            };
+            state.caches.emplace(uri, cache);
         }
         return getCache(state, uri);
     }
 
-    void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
+public:
+    int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
     {
-        retrySQLite<void>([&]() {
+        return retrySQLite<int>([&]() {
             auto state(_state.lock());
             SQLiteTxn txn(state->db);
 
@@ -190,17 +197,29 @@ public:
             auto cache(queryCacheRaw(*state, uri));
 
             if (cache)
-                return;
+                return cache->id;
 
-            state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
+            Cache ret {
-            assert(sqlite3_changes(state->db) == 1);
+                .id = -1, // set below
-            state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+                .storeDir = storeDir,
+                .wantMassQuery = wantMassQuery,
+                .priority = priority,
+            };
+
+            {
+                auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority));
+                assert(r.next());
+                ret.id = (int) r.getInt(0);
+            }
+
+            state->caches[uri] = ret;
 
             txn.commit();
+            return ret.id;
         });
     }
 
-    std::optional<CacheInfo> cacheExists(const std::string & uri) override
+    std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override
     {
         return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
             auto state(_state.lock());
@@ -208,6 +227,7 @@ public:
             if (!cache)
                 return std::nullopt;
             return CacheInfo {
+                .id = cache->id,
                 .wantMassQuery = cache->wantMassQuery,
                 .priority = cache->priority
             };
@@ -371,4 +391,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache()
     return cache;
 }
 
+ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath)
+{
+    return make_ref<NarInfoDiskCacheImpl>(dbPath);
+}
+
 }

src/libstore/nar-info-disk-cache.hh

@@ -13,16 +13,17 @@ public:
 
     virtual ~NarInfoDiskCache() { }
 
-    virtual void createCache(const std::string & uri, const Path & storeDir,
+    virtual int createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
 
     struct CacheInfo
     {
+        int id;
         bool wantMassQuery;
         int priority;
     };
 
-    virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
+    virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0;
 
     virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
         const std::string & uri, const std::string & hashPart) = 0;
@@ -45,4 +46,6 @@ public:
    multiple threads. */
 ref<NarInfoDiskCache> getNarInfoDiskCache();
 
+ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);
+
 }

src/libstore/nix-store.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil
-Cflags: -I${includedir}/nix -std=c++20
+Cflags: -I${includedir}/nix -std=c++2a

src/libstore/profiles.cc

@@ -282,7 +282,7 @@ std::string optimisticLockProfile(const Path & profile)
 
 Path profilesDir()
 {
-    auto profileRoot = getDataDir() + "/nix/profiles";
+    auto profileRoot = createNixStateDir() + "/profiles";
     createDirs(profileRoot);
     return profileRoot;
 }
@@ -290,7 +290,7 @@ Path profilesDir()
 
 Path getDefaultProfile()
 {
-    Path profileLink = getHome() + "/.nix-profile";
+    Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile";
     try {
         auto profile =
             getuid() == 0

src/libstore/profiles.hh

@@ -72,8 +72,9 @@ std::string optimisticLockProfile(const Path & profile);
    profiles. */
 Path profilesDir();
 
-/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
+/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/
-   it. */
+   nix/profile if XDG Base Directory Support is enabled), and create if doesn't
+   exist */
 Path getDefaultProfile();
 
 }

src/libstore/remote-store.cc

@@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn)
         overrides.erase(settings.useSubstitutes.name);
         overrides.erase(loggerSettings.showTrace.name);
         overrides.erase(settings.experimentalFeatures.name);
+        overrides.erase(settings.pluginFiles.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;

src/libstore/s3-binary-cache-store.cc

@@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
 
     void init() override
     {
-        if (auto cacheInfo = diskCache->cacheExists(getUri())) {
+        if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
             wantMassQuery.setDefault(cacheInfo->wantMassQuery);
             priority.setDefault(cacheInfo->priority);
         } else {

src/libstore/sqlite.cc

@@ -41,6 +41,15 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex
         throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf));
 }
 
+static void traceSQL(void * x, const char * sql)
+{
+    // wacky delimiters:
+    //   so that we're quite unambiguous without escaping anything
+    // notice instead of trace:
+    //   so that this can be enabled without getting the firehose in our face.
+    notice("SQL<[%1%]>", sql);
+};
+
 SQLite::SQLite(const Path & path, bool create)
 {
     // useSQLiteWAL also indicates what virtual file system we need.  Using
@@ -58,6 +67,11 @@ SQLite::SQLite(const Path & path, bool create)
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
         SQLiteError::throw_(db, "setting timeout");
 
+    if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") {
+        // To debug sqlite statements; trace all of them
+        sqlite3_trace(db, &traceSQL, nullptr);
+    }
+
     exec("pragma foreign_keys = 1");
 }
 

src/libstore/tests/derivation.cc

@@ -0,0 +1,124 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "derivations.hh"
+
+#include "tests/libstore.hh"
+
+namespace nix {
+
+class DerivationTest : public LibStoreTest
+{
+};
+
+#define TEST_JSON(TYPE, NAME, STR, VAL, ...)                           \
+    TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) {            \
+        using nlohmann::literals::operator "" _json;                   \
+        ASSERT_EQ(                                                     \
+            STR ## _json,                                              \
+            (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__));  \
+    }
+
+TEST_JSON(DerivationOutput, inputAddressed,
+    R"({
+        "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
+    })",
+    (DerivationOutput::InputAddressed {
+        .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"),
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationOutput, caFixed,
+    R"({
+        "hashAlgo": "r:sha256",
+        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+        "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
+    })",
+    (DerivationOutput::CAFixed {
+        .hash = {
+            .method = FileIngestionMethod::Recursive,
+            .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
+        },
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationOutput, caFloating,
+    R"({
+        "hashAlgo": "r:sha256"
+    })",
+    (DerivationOutput::CAFloating {
+        .method = FileIngestionMethod::Recursive,
+        .hashType = htSHA256,
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationOutput, deferred,
+    R"({ })",
+    DerivationOutput::Deferred { },
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationOutput, impure,
+    R"({
+        "hashAlgo": "r:sha256",
+        "impure": true
+    })",
+    (DerivationOutput::Impure {
+        .method = FileIngestionMethod::Recursive,
+        .hashType = htSHA256,
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(Derivation, impure,
+    R"({
+      "inputSrcs": [
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+      ],
+      "inputDrvs": {
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [
+          "cat",
+          "dog"
+        ]
+      },
+      "system": "wasm-sel4",
+      "builder": "foo",
+      "args": [
+        "bar",
+        "baz"
+      ],
+      "env": {
+        "BIG_BAD": "WOLF"
+      },
+      "outputs": {}
+    })",
+    ({
+        Derivation drv;
+        drv.inputSrcs = {
+            store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
+        };
+        drv.inputDrvs = {
+            {
+                store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
+                {
+                    "cat",
+                    "dog",
+                },
+            }
+        };
+        drv.platform = "wasm-sel4";
+        drv.builder = "foo";
+        drv.args = {
+            "bar",
+            "baz",
+        };
+        drv.env = {
+            {
+                "BIG_BAD",
+                "WOLF",
+            },
+        };
+        drv;
+    }))
+
+#undef TEST_JSON
+
+}

src/libstore/tests/nar-info-disk-cache.cc

@@ -0,0 +1,123 @@
+#include "nar-info-disk-cache.hh"
+
+#include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
+#include "sqlite.hh"
+#include <sqlite3.h>
+
+
+namespace nix {
+
+TEST(NarInfoDiskCacheImpl, create_and_read) {
+    // This is a large single test to avoid some setup overhead.
+
+    int prio = 12345;
+    bool wantMassQuery = true;
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir);
+    Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite");
+
+    int savedId;
+    int barId;
+    SQLite db;
+    SQLiteStmt getIds;
+
+    {
+        auto cache = getTestNarInfoDiskCache(dbPath);
+
+        // Set up "background noise" and check that different caches receive different ids
+        {
+            auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio);
+            auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12);
+            ASSERT_NE(bc1, bc2);
+            barId = bc1;
+        }
+
+        // Check that the fields are saved and returned correctly. This does not test
+        // the select statement yet, because of in-memory caching.
+        savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);;
+        {
+            auto r = cache->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(savedId, r->id);
+        }
+
+        // We're going to pay special attention to the id field because we had a bug
+        // that changed it.
+        db = SQLite(dbPath);
+        getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
+
+        {
+            auto q(getIds.use());
+            ASSERT_TRUE(q.next());
+            ASSERT_EQ(savedId, q.getInt(0));
+            ASSERT_FALSE(q.next());
+        }
+
+        // Pretend that the caches are older, but keep one up to date, as "background noise"
+        db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';");
+
+        // This shows that the in-memory cache works
+        {
+            auto r = cache->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+        }
+    }
+
+    {
+        // We can't clear the in-memory cache, so we use a new cache object. This is
+        // more realistic anyway.
+        auto cache2 = getTestNarInfoDiskCache(dbPath);
+
+        {
+            auto r = cache2->upToDateCacheExists("http://foo");
+            ASSERT_FALSE(r);
+        }
+
+        // "Update", same data, check that the id number is reused
+        cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);
+
+        {
+            auto r = cache2->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(r->id, savedId);
+        }
+
+        {
+            auto q(getIds.use());
+            ASSERT_TRUE(q.next());
+            auto currentId = q.getInt(0);
+            ASSERT_FALSE(q.next());
+            ASSERT_EQ(currentId, savedId);
+        }
+
+        // Check that the fields can be modified, and the id remains the same
+        {
+            auto r0 = cache2->upToDateCacheExists("https://bar");
+            ASSERT_FALSE(r0);
+
+            cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10);
+            auto r = cache2->upToDateCacheExists("https://bar");
+            ASSERT_EQ(r->wantMassQuery, !wantMassQuery);
+            ASSERT_EQ(r->priority, prio + 10);
+            ASSERT_EQ(r->id, barId);
+        }
+
+        // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp)
+        // {
+        //     cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20);
+        //     auto r = cache2->upToDateCacheExists("https://bar");
+        //     ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+        //     ASSERT_EQ(r->priority, prio + 20);
+        // }
+    }
+}
+
+}

src/libutil/args.cc

@@ -29,7 +29,15 @@ void Args::removeFlag(const std::string & longName)
 
 void Completions::add(std::string completion, std::string description)
 {
-    assert(description.find('\n') == std::string::npos);
+    description = trim(description);
+    // ellipsize overflowing content on the back of the description
+    auto end_index = description.find_first_of(".\n");
+    if (end_index != std::string::npos) {
+        auto needs_ellipsis = end_index != description.size() - 1;
+        description.resize(end_index);
+        if (needs_ellipsis)
+            description.append(" [...]");
+    }
     insert(Completion {
         .completion = completion,
         .description = description

src/libutil/namespaces.cc

@@ -0,0 +1,97 @@
+#if __linux__
+
+#include "namespaces.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <sys/mount.h>
+
+namespace nix {
+
+bool userNamespacesSupported()
+{
+    static auto res = [&]() -> bool
+    {
+        if (!pathExists("/proc/self/ns/user")) {
+            debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y");
+            return false;
+        }
+
+        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+        if (!pathExists(maxUserNamespaces) ||
+            trim(readFile(maxUserNamespaces)) == "0")
+        {
+            debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'");
+            return false;
+        }
+
+        Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone";
+        if (pathExists(procSysKernelUnprivilegedUsernsClone)
+            && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0")
+        {
+            debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'");
+            return false;
+        }
+
+        try {
+            Pid pid = startProcess([&]()
+            {
+                _exit(0);
+            }, {
+                .cloneFlags = CLONE_NEWUSER
+            });
+
+            auto r = pid.wait();
+            assert(!r);
+        } catch (SysError & e) {
+            debug("user namespaces do not work on this system: %s", e.msg());
+            return false;
+        }
+
+        return true;
+    }();
+    return res;
+}
+
+bool mountAndPidNamespacesSupported()
+{
+    static auto res = [&]() -> bool
+    {
+        try {
+
+            Pid pid = startProcess([&]()
+            {
+                /* Make sure we don't remount the parent's /proc. */
+                if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
+                    _exit(1);
+
+                /* Test whether we can remount /proc. The kernel disallows
+                   this if /proc is not fully visible, i.e. if there are
+                   filesystems mounted on top of files inside /proc.  See
+                   https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */
+                if (mount("none", "/proc", "proc", 0, 0) == -1)
+                    _exit(2);
+
+                _exit(0);
+            }, {
+                .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0)
+            });
+
+            if (pid.wait()) {
+                debug("PID namespaces do not work on this system: cannot remount /proc");
+                return false;
+            }
+
+        } catch (SysError & e) {
+            debug("mount namespaces do not work on this system: %s", e.msg());
+            return false;
+        }
+
+        return true;
+    }();
+    return res;
+}
+
+}
+
+#endif

src/libutil/namespaces.hh

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace nix {
+
+#if __linux__
+
+bool userNamespacesSupported();
+
+bool mountAndPidNamespacesSupported();
+
+#endif
+
+}

src/libutil/tests/url.cc

@@ -302,4 +302,37 @@ namespace nix {
         ASSERT_EQ(d, s);
     }
 
+
+    /* ----------------------------------------------------------------------------
+     * percentEncode
+     * --------------------------------------------------------------------------*/
+
+    TEST(percentEncode, encodesUrlEncodedString) {
+        std::string s = percentEncode("==@==");
+        std::string d = "%3D%3D%40%3D%3D";
+        ASSERT_EQ(d, s);
+    }
+
+    TEST(percentEncode, keepArgument) {
+        std::string a = percentEncode("abd / def");
+        std::string b = percentEncode("abd / def", "/");
+        ASSERT_EQ(a, "abd%20%2F%20def");
+        ASSERT_EQ(b, "abd%20/%20def");
+    }
+
+    TEST(percentEncode, inverseOfDecode) {
+        std::string original = "%3D%3D%40%3D%3D";
+        std::string once = percentEncode(original);
+        std::string back = percentDecode(once);
+
+        ASSERT_EQ(back, original);
+    }
+
+    TEST(percentEncode, trailingPercent) {
+        std::string s = percentEncode("==@==%");
+        std::string d = "%3D%3D%40%3D%3D%25";
+
+        ASSERT_EQ(d, s);
+    }
+
 }

src/libutil/url.cc

@@ -88,17 +88,22 @@ std::map<std::string, std::string> decodeQuery(const std::string & query)
     return result;
 }
 
-std::string percentEncode(std::string_view s)
+const static std::string allowedInQuery = ":@/?";
+const static std::string allowedInPath = ":@/";
+
+std::string percentEncode(std::string_view s, std::string_view keep)
 {
     std::string res;
     for (auto & c : s)
+        // unreserved + keep
         if ((c >= 'a' && c <= 'z')
             || (c >= 'A' && c <= 'Z')
             || (c >= '0' && c <= '9')
-            || strchr("-._~!$&'()*+,;=:@", c))
+            || strchr("-._~", c)
+            || keep.find(c) != std::string::npos)
             res += c;
         else
-            res += fmt("%%%02x", (unsigned int) c);
+            res += fmt("%%%02X", (unsigned int) c);
     return res;
 }
 
@@ -109,9 +114,9 @@ std::string encodeQuery(const std::map<std::string, std::string> & ss)
     for (auto & [name, value] : ss) {
         if (!first) res += '&';
         first = false;
-        res += percentEncode(name);
+        res += percentEncode(name, allowedInQuery);
         res += '=';
-        res += percentEncode(value);
+        res += percentEncode(value, allowedInQuery);
     }
     return res;
 }
@@ -122,7 +127,7 @@ std::string ParsedURL::to_string() const
         scheme
         + ":"
         + (authority ? "//" + *authority : "")
-        + path
+        + percentEncode(path, allowedInPath)
         + (query.empty() ? "" : "?" + encodeQuery(query))
         + (fragment.empty() ? "" : "#" + percentEncode(fragment));
 }

src/libutil/url.hh

@@ -22,6 +22,7 @@ struct ParsedURL
 MakeError(BadURL, Error);
 
 std::string percentDecode(std::string_view in);
+std::string percentEncode(std::string_view s, std::string_view keep="");
 
 std::map<std::string, std::string> decodeQuery(const std::string & query);
 

src/libutil/util.cc

@@ -36,6 +36,7 @@
 #ifdef __linux__
 #include <sys/prctl.h>
 #include <sys/resource.h>
+#include <sys/mman.h>
 
 #include <cmath>
 #endif
@@ -608,6 +609,19 @@ Path getDataDir()
     return dataDir ? *dataDir : getHome() + "/.local/share";
 }
 
+Path getStateDir()
+{
+    auto stateDir = getEnv("XDG_STATE_HOME");
+    return stateDir ? *stateDir : getHome() + "/.local/state";
+}
+
+Path createNixStateDir()
+{
+    Path dir = getStateDir() + "/nix";
+    createDirs(dir);
+    return dir;
+}
+
 
 std::optional<Path> getSelfExe()
 {
@@ -1051,9 +1065,17 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun)
 }
 
 
+static int childEntry(void * arg)
+{
+    auto main = (std::function<void()> *) arg;
+    (*main)();
+    return 1;
+}
+
+
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 {
-    auto wrapper = [&]() {
+    std::function<void()> wrapper = [&]() {
         if (!options.allowVfork)
             logger = makeSimpleLogger();
         try {
@@ -1073,7 +1095,27 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
             _exit(1);
     };
 
-    pid_t pid = doFork(options.allowVfork, wrapper);
+    pid_t pid = -1;
+
+    if (options.cloneFlags) {
+        #ifdef __linux__
+        // Not supported, since then we don't know when to free the stack.
+        assert(!(options.cloneFlags & CLONE_VM));
+
+        size_t stackSize = 1 * 1024 * 1024;
+        auto stack = (char *) mmap(0, stackSize,
+            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+        if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+        Finally freeStack([&]() { munmap(stack, stackSize); });
+
+        pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
+        #else
+        throw Error("clone flags are only supported on Linux");
+        #endif
+    } else
+        pid = doFork(options.allowVfork, wrapper);
+
     if (pid == -1) throw SysError("unable to fork");
 
     return pid;

src/libutil/util.hh

@@ -158,6 +158,12 @@ Path getDataDir();
 /* Return the path of the current executable. */
 std::optional<Path> getSelfExe();
 
+/* Return $XDG_STATE_HOME or $HOME/.local/state. */
+Path getStateDir();
+
+/* Create the Nix state directory and return the path to it. */
+Path createNixStateDir();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
@@ -301,6 +307,7 @@ struct ProcessOptions
     bool dieWithParent = true;
     bool runExitHandlers = false;
     bool allowVfork = false;
+    int cloneFlags = 0; // use clone() with the specified flags (Linux only)
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());

src/nix-build/nix-build.cc

@@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv)
                 "SHELL=%5%; "
                 "BASH=%5%; "
                 "set +e; "
-                R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
+                R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" +
+                (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s"
+                               : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") +
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "

src/nix-channel/nix-channel.cc

@@ -164,8 +164,8 @@ static int main_nix_channel(int argc, char ** argv)
     {
         // Figure out the name of the `.nix-channels' file to use
         auto home = getHome();
-        channelsList = home + "/.nix-channels";
+        channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels";
-        nixDefExpr = home + "/.nix-defexpr";
+        nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr";
 
         // Figure out the name of the channels profile.
         profile = profilesDir() + "/channels";

src/nix-env/nix-env.cc

@@ -1289,7 +1289,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs)
         throw UsageError("exactly one argument expected");
 
     Path profile = absPath(opArgs.front());
-    Path profileLink = getHome() + "/.nix-profile";
+    Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile";
 
     switchLink(profileLink, profile);
 }
@@ -1393,7 +1393,10 @@ static int main_nix_env(int argc, char * * argv)
         Globals globals;
 
         globals.instSource.type = srcUnknown;
-        globals.instSource.nixExprPath = getHome() + "/.nix-defexpr";
+        {
+            Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr";
+            globals.instSource.nixExprPath = nixExprPath;
+        }
         globals.instSource.systemFilter = "*";
 
         if (!pathExists(globals.instSource.nixExprPath)) {

src/nix/app.cc

@@ -1,4 +1,5 @@
 #include "installables.hh"
+#include "installable-derived-path.hh"
 #include "store-api.hh"
 #include "eval-inline.hh"
 #include "eval-cache.hh"
@@ -8,30 +9,6 @@
 
 namespace nix {
 
-struct InstallableDerivedPath : Installable
-{
-    ref<Store> store;
-    const DerivedPath derivedPath;
-
-    InstallableDerivedPath(ref<Store> store, const DerivedPath & derivedPath)
-        : store(store)
-        , derivedPath(derivedPath)
-    {
-    }
-
-    std::string what() const override { return derivedPath.to_string(*store); }
-
-    DerivedPathsWithInfo toDerivedPaths() override
-    {
-        return {{derivedPath}};
-    }
-
-    std::optional<StorePath> getStorePath() override
-    {
-        return std::nullopt;
-    }
-};
-
 /**
  * Return the rewrites that are needed to resolve a string whose context is
  * included in `dependencies`.
@@ -146,7 +123,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store)
 
     for (auto & ctxElt : unresolved.context)
         installableContext.push_back(
-            std::make_shared<InstallableDerivedPath>(store, ctxElt));
+            std::make_shared<InstallableDerivedPath>(store, DerivedPath { ctxElt }));
 
     auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext);
     res.program = resolveString(*store, unresolved.program, builtContext);

src/nix/bundle.cc

@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "installable-flake.hh"
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"

src/nix/daemon.cc

@@ -34,6 +34,43 @@
 using namespace nix;
 using namespace nix::daemon;
 
+struct AuthorizationSettings : Config {
+
+    Setting<Strings> trustedUsers{
+        this, {"root"}, "trusted-users",
+        R"(
+          A list of names of users (separated by whitespace) that have
+          additional rights when connecting to the Nix daemon, such as the
+          ability to specify additional binary caches, or to import unsigned
+          NARs. You can also specify groups by prefixing them with `@`; for
+          instance, `@wheel` means all users in the `wheel` group. The default
+          is `root`.
+
+          > **Warning**
+          >
+          > Adding a user to `trusted-users` is essentially equivalent to
+          > giving that user root access to the system. For example, the user
+          > can set `sandbox-paths` and thereby obtain read access to
+          > directories that are otherwise inacessible to them.
+        )"};
+
+    /* ?Who we trust to use the daemon in safe ways */
+    Setting<Strings> allowedUsers{
+        this, {"*"}, "allowed-users",
+        R"(
+          A list of names of users (separated by whitespace) that are allowed
+          to connect to the Nix daemon. As with the `trusted-users` option,
+          you can specify groups by prefixing them with `@`. Also, you can
+          allow all users by specifying `*`. The default is `*`.
+
+          Note that trusted users are always allowed to connect.
+        )"};
+};
+
+AuthorizationSettings authorizationSettings;
+
+static GlobalConfig::Register rSettings(&authorizationSettings);
+
 #ifndef __linux__
 #define SPLICE_F_MOVE 0
 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags)
@@ -203,8 +240,8 @@ static void daemonLoop()
             struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0;
             std::string group = gr ? gr->gr_name : std::to_string(peer.gid);
 
-            Strings trustedUsers = settings.trustedUsers;
+            Strings trustedUsers = authorizationSettings.trustedUsers;
-            Strings allowedUsers = settings.allowedUsers;
+            Strings allowedUsers = authorizationSettings.allowedUsers;
 
             if (matchUser(user, group, trustedUsers))
                 trusted = Trusted;
@@ -241,14 +278,7 @@ static void daemonLoop()
                 //  Handle the connection.
                 FdSource from(remote.get());
                 FdSink to(remote.get());
-                processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) {
+                processConnection(openUncachedStore(), from, to, trusted, NotRecursive);
-#if 0
-                    /* Prevent users from doing something very dangerous. */
-                    if (geteuid() == 0 &&
-                        querySetting("build-users-group", "") == "")
-                        throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-                });
 
                 exit(0);
             }, options);
@@ -301,7 +331,7 @@ static void runDaemon(bool stdio)
             /* Auth hook is empty because in this mode we blindly trust the
                standard streams. Limiting access to those is explicitly
                not `nix-daemon`'s responsibility. */
-            processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
+            processConnection(openUncachedStore(), from, to, Trusted, NotRecursive);
         }
     } else
         daemonLoop();

src/nix/develop.cc

@@ -1,5 +1,6 @@
 #include "eval.hh"
 #include "command.hh"
+#include "installable-flake.hh"
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"

src/nix/diff-closures.cc

@@ -106,7 +106,7 @@ void printClosureDiff(
 
 using namespace nix;
 
-struct CmdDiffClosures : SourceExprCommand
+struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions
 {
     std::string _before, _after;
 

src/nix/edit.cc

@@ -3,6 +3,7 @@
 #include "eval.hh"
 #include "attr-path.hh"
 #include "progress-bar.hh"
+#include "editor-for.hh"
 
 #include <unistd.h>
 

src/nix/eval.cc

@@ -11,13 +11,13 @@
 
 using namespace nix;
 
-struct CmdEval : MixJSON, InstallableCommand
+struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption
 {
     bool raw = false;
     std::optional<std::string> apply;
     std::optional<Path> writeTo;
 
-    CmdEval() : InstallableCommand(true /* supportReadOnlyMode */)
+    CmdEval() : InstallableCommand()
     {
         addFlag({
             .longName = "raw",

src/nix/flake.cc

@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "installable-flake.hh"
 #include "common-args.hh"
 #include "shared.hh"
 #include "eval.hh"