Merge branch 'master' into debug-exploratory-PR

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,7 @@
+**Release Notes**
+Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed.
+
+
+**Testing**
+
+If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).

.github/workflows/hydra_status.yml

@@ -0,0 +1,16 @@
+name: Hydra status
+on:
+  schedule:
+    - cron: "12,42 * * * *"
+  workflow_dispatch:
+jobs:
+  check_hydra_status:
+    name: Check Hydra status
+    if: github.repository_owner == 'NixOS'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - run: bash scripts/check-hydra-status.sh
+

.version

@@ -1 +1 @@
-2.5
+2.6.0

configure.ac

@@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
     [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
 ])
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
 # Look for libcpuid.
+have_libcpuid=
 if test "$machine_name" = "x86_64"; then
-  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
+    AC_ARG_ENABLE([cpuid],
-  have_libcpuid=1
+                  AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)]))
-  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+    if test "x$enable_cpuid" != "xno"; then
+      PKG_CHECK_MODULES([LIBCPUID], [libcpuid],
+        [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"
+         have_libcpuid=1
+         AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])]
+      )
+    fi
 fi
 AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 

doc/manual/generate-options.nix

@@ -8,17 +8,19 @@ concatStrings (map
     let option = options.${name}; in
     "  - `${name}`  \n\n"
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
-    + "    **Default:** " + (
+    + (if option.documentDefault
-      if option.value == "" || option.value == []
+       then "    **Default:** " + (
-      then "*empty*"
+       if option.value == "" || option.value == []
-      else if isBool option.value
+       then "*empty*"
-      then (if option.value then "`true`" else "`false`")
+       else if isBool option.value
-      else
+       then (if option.value then "`true`" else "`false`")
-        # n.b. a StringMap value type is specified as a string, but
+       else
-        # this shows the value type.  The empty stringmap is "null" in
+         # n.b. a StringMap value type is specified as a string, but
-        # JSON, but that converts to "{ }" here.
+         # this shows the value type.  The empty stringmap is "null" in
-        (if isAttrs option.value then "`\"\"`"
+         # JSON, but that converts to "{ }" here.
-         else "`" + toString option.value + "`")) + "\n\n"
+         (if isAttrs option.value then "`\"\"`"
+          else "`" + toString option.value + "`")) + "\n\n"
+       else "    **Default:** *machine-specific*")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

doc/manual/local.mk

@@ -12,11 +12,13 @@ man-pages := $(foreach n, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
-	NIX_STATE_DIR=/dummy
+	NIX_STATE_DIR=/dummy \
+	NIX_CONFIG='cores = 0'
 
 nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 

doc/manual/src/SUMMARY.md.in

@@ -72,6 +72,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for
 $ uname
 Linux
 
-$ nix build \
+$ nix build --impure \
-  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
 [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
 

doc/manual/src/contributing/hacking.md

@@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found:
 $ nix-shell
 ```
 
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop
+```
+
+To get a shell with a different compilation environment (e.g. stdenv,
+gccStdenv, clangStdenv, clang11Stdenv):
+
+```console
+$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
+```
+
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop .#clang11StdenvPackages
+```
+
 To build Nix itself in this shell:
 
 ```console

doc/manual/src/glossary.md

@@ -47,7 +47,7 @@
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
     has a reference.
-    
+
     A derivation can reference other derivations and sources (but not
     output paths), whereas an output path only references other output
     paths.
@@ -66,7 +66,7 @@
     is necessary to deploy whole closures, since otherwise at runtime
     files could be missing. The command `nix-store -qR` prints out
     closures of store paths.
-    
+
     As an example, if the store object at path `P` contains a reference
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
@@ -98,3 +98,7 @@
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
     and `nix-store --restore`.
+  - `∅` \
+    The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
+  - `ε` \
+    The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.

doc/manual/src/installation/installing-binary.md

@@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble:
 - update `/etc/synthetic.conf` to direct macOS to create a "synthetic"
   empty root directory to mount your volume
 - specify mount options for the volume in `/etc/fstab`
+  - `rw`: read-write
+  - `noauto`: prevent the system from auto-mounting the volume (so the
+    LaunchDaemon mentioned below can control mounting it, and to avoid
+    masking problems with that mounting service).
+  - `nobrowse`: prevent the Nix Store volume from showing up on your
+    desktop; also keeps Spotlight from spending resources to index
+    this volume
+  <!-- TODO:
+  - `suid`: honor setuid? surely not? ...
+  - `owners`: honor file ownership on the volume
+
+    For now I'll avoid pretending to understand suid/owners more
+    than I do. There've been some vague reports of file-ownership
+    and permission issues, particularly in cloud/VM/headless setups.
+    My pet theory is that this has something to do with these setups
+    not having a token that gets delegated to initial/admin accounts
+    on macOS. See scripts/create-darwin-volume.sh for a little more.
+
+    In any case, by Dec 4 2021, it _seems_ like some combination of
+    suid, owners, and calling diskutil enableOwnership have stopped
+    new reports from coming in. But I hesitate to celebrate because we
+    haven't really named and catalogued the behavior, understood what
+    we're fixing, and validated that all 3 components are essential.
+  -->
 - if you have FileVault enabled
     - generate an encryption password
     - put it in your system Keychain

doc/manual/src/installation/installing-docker.md

@@ -3,7 +3,7 @@
 To run the latest stable release of Nix with Docker run the following command:
 
 ```console
-$ docker -ti run nixos/nix
+$ docker run -ti nixos/nix
 Unable to find image 'nixos/nix:latest' locally
 latest: Pulling from nixos/nix
 5843afab3874: Pull complete
@@ -16,7 +16,7 @@ nix (Nix) 2.3.12
 35ca4ada6e96:/# exit
 ```
 
-# What is included in Nix' Docker image?
+# What is included in Nix's Docker image?
 
 The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
 (and not with `Dockerfile` as it is usual with Docker images). You can still
@@ -54,6 +54,6 @@ You can also build a Docker image from source yourself:
 
 ```console
 $ nix build ./\#hydraJobs.dockerImage.x86_64-linux
-$ docker load -i ./result
+$ docker load -i ./result/image.tar.gz
 $ docker run -ti nix:2.5pre20211105
 ```

doc/manual/src/installation/prerequisites-source.md

@@ -44,6 +44,11 @@
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
+  - The `libsodium` library for verifying cryptographic signatures
+    of contents fetched from binary caches.
+    It can be obtained from the official web site
+    <https://libsodium.org>.
+
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from
@@ -58,3 +63,11 @@
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
+
+  - On 64-bit x86 machines only, `libcpuid` library
+    is used to determine which microarchitecture levels are supported
+    (e.g., as whether to have `x86_64-v2-linux` among additional system types).
+    The library is available from its homepage
+    <http://libcpuid.sourceforge.net>.
+    This is an optional dependency and can be disabled
+    by providing a `--disable-cpuid` to the `configure` script.

doc/manual/src/installation/supported-platforms.md

@@ -4,4 +4,4 @@ Nix is currently supported on the following platforms:
 
   - Linux (i686, x86\_64, aarch64).
 
-  - macOS (x86\_64).
+  - macOS (x86\_64, aarch64).

doc/manual/src/package-management/basic-package-mgmt.md

@@ -40,7 +40,7 @@ $ nix-channel --update
 > 
 > On NixOS, you’re automatically subscribed to a NixOS channel
 > corresponding to your NixOS major release (e.g.
-> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
 > to the Nixpkgs channel, except that it contains only Linux binaries
 > and is updated only if a set of regression tests succeed.
 

doc/manual/src/release-notes/rl-2.4.md

@@ -395,6 +395,7 @@ dramforever,
 Dustin DeWeese,
 edef,
 Eelco Dolstra,
+Ellie Hermaszewska,
 Emilio Karakey,
 Emily,
 Eric Culp,
@@ -405,7 +406,7 @@ Federico Pellegrin,
 Finn Behrens,
 Florian Franzen,
 Félix Baylac-Jacqué,
-Gabriel Gonzalez,
+Gabriella Gonzalez,
 Geoff Reedy,
 Georges Dubus,
 Graham Christensen,
@@ -428,7 +429,6 @@ Jaroslavas Pocepko,
 Jarrett Keifer,
 Jeremy Schlatter,
 Joachim Breitner,
-Joe Hermaszewski,
 Joe Pea,
 John Ericson,
 Jonathan Ringer,

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,16 @@
+# Release 2.5 (2021-12-13)
+
+* The garbage collector no longer blocks new builds, so the message
+  `waiting for the big garbage collector lock...` is a thing of the
+  past.
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.
+
+* New built-in function: `builtins.groupBy`, with the same functionality as
+  Nixpkgs' `lib.groupBy`, but faster.
+
+* `nix repl` now has a `:log` command.

doc/manual/src/release-notes/rl-next.md

@@ -1,7 +1,6 @@
-# Release 2.5 (2021-XX-XX)
+# Release X.Y (202?-??-??)
 
-* Binary cache stores now have a setting `compression-level`.
+* The TOML parser used by `builtins.fromTOML` has been replaced by [a
-
+  more compliant one](https://github.com/ToruNiina/toml11).
-* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+* Added `:st`/`:show-trace` commands to nix repl, which are used to
-
+  set or toggle display of error traces.
-* Lists can now be compared lexicographically using the `<` operator.

docker.nix

@@ -137,11 +137,8 @@ let
         name = "root-profile-env";
         paths = defaultPkgs;
       };
-      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
-        mkdir $out
+        cat > $out <<EOF
-        cp -a ${rootEnv}/* $out/
-
-        cat > $out/manifest.nix <<EOF
         [
         ${lib.concatStringsSep "\n" (builtins.map (drv: let
           outputs = drv.outputsToInstall or [ "out" ];
@@ -161,6 +158,11 @@ let
         ]
         EOF
       '';
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+        ln -s ${manifest} $out/manifest.nix
+      '';
     in
     pkgs.runCommand "base-system"
       {
@@ -178,6 +180,9 @@ let
       set -x
       mkdir -p $out/etc
 
+      mkdir -p $out/etc/ssl/certs
+      ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
+
       cat $passwdContentsPath > $out/etc/passwd
       echo "" >> $out/etc/passwd
 
@@ -227,6 +232,9 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
     rm -rf nix-support
     ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
   '';
+  fakeRootCommands = ''
+    chmod 1777 tmp
+  '';
 
   config = {
     Cmd = [ "/root/.nix-profile/bin/bash" ];

flake.nix

@@ -22,15 +22,36 @@
 
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
+      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      forAllSystemsAndStdenvs = f: forAllSystems (system:
+        nixpkgs.lib.listToAttrs
+          (map
+            (n:
+            nixpkgs.lib.nameValuePair "${n}Packages" (
+              f system n
+            )) stdenvs
+          )
+      );
+
+      forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems (system:
+      nixpkgsFor =
-        import nixpkgs {
+        let stdenvsPackages = forAllSystemsAndStdenvs
-          inherit system;
+          (system: stdenv:
-          overlays = [ self.overlay ];
+            import nixpkgs {
-        }
+              inherit system;
-      );
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
+            }
+          );
+        in
+        # Add the `stdenvPackages` at toplevel, both because these are the ones
+        # we want most of the time and for backwards compatibility
+        forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
 
       commonDeps = pkgs: with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
@@ -75,7 +96,7 @@
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
-            buildPackages.pkgconfig
+            buildPackages.pkg-config
 
             # Tests
             buildPackages.git
@@ -255,18 +276,15 @@
             $(cat ${installerClosureInfo}/store-paths)
         '';
 
-    in {
+      overlayFor = getStdenv: final: prev:
-
+      let currentStdenv = getStdenv final; in
-      # A Nixpkgs overlay that overrides the 'nix' and
+      {
-      # 'nix.perl-bindings' packages.
-      overlay = final: prev: {
-
         nixStable = prev.nix;
 
         # Forward from the previous stage as we don’t want it to pick the lowdown override
         nixUnstable = prev.nixUnstable;
 
-        nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
+        nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
 
@@ -288,9 +306,9 @@
               mkdir -p $out/lib
               cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
               rm -f $out/lib/*.a
-              ${lib.optionalString stdenv.isLinux ''
+              ${lib.optionalString currentStdenv.isLinux ''
                 chmod u+w $out/lib/*.so.*
-                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+                patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
               ''}
             '';
 
@@ -317,7 +335,7 @@
 
           strictDeps = true;
 
-          passthru.perl-bindings = with final; stdenv.mkDerivation {
+          passthru.perl-bindings = with final; currentStdenv.mkDerivation {
             name = "nix-perl-${version}";
 
             src = self;
@@ -325,7 +343,7 @@
             nativeBuildInputs =
               [ buildPackages.autoconf-archive
                 buildPackages.autoreconfHook
-                buildPackages.pkgconfig
+                buildPackages.pkg-config
               ];
 
             buildInputs =
@@ -336,8 +354,8 @@
                 pkgs.perl
                 boost
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+              ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
-              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+              ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -351,7 +369,7 @@
 
         };
 
-        lowdown-nix = with final; stdenv.mkDerivation rec {
+        lowdown-nix = with final; currentStdenv.mkDerivation rec {
           name = "lowdown-0.9.0";
 
           src = lowdown-src;
@@ -361,15 +379,20 @@
           nativeBuildInputs = [ buildPackages.which ];
 
           configurePhase = ''
-              ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
+              ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
               ./configure \
                 PREFIX=${placeholder "dev"} \
                 BINDIR=${placeholder "bin"}/bin
-            '';
+          '';
         };
-
       };
 
+    in {
+
+      # A Nixpkgs overlay that overrides the 'nix' and
+      # 'nix.perl-bindings' packages.
+      overlay = overlayFor (p: p.stdenv);
+
       hydraJobs = {
 
         # Binary package for various platforms.
@@ -610,15 +633,22 @@
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
         };
-      }) crossSystems)));
+      }) crossSystems)) // (builtins.listToAttrs (map (stdenvName:
+      nixpkgsFor.${system}.lib.nameValuePair
+        "nix-${stdenvName}"
+        nixpkgsFor.${system}."${stdenvName}Packages".nix
+      ) stdenvs))
+      );
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 
-      devShell = forAllSystems (system:
+      devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages);
+
+      devShells = forAllSystemsAndStdenvs (system: stdenv:
         with nixpkgsFor.${system};
         with commonDeps pkgs;
 
-        stdenv.mkDerivation {
+        nixpkgsFor.${system}.${stdenv}.mkDerivation {
           name = "nix";
 
           outputs = [ "out" "dev" "doc" ];
@@ -637,6 +667,9 @@
               PATH=$prefix/bin:$PATH
               unset PYTHONPATH
               export MANPATH=$out/share/man:$MANPATH
+
+              # Make bash completion work.
+              XDG_DATA_DIRS+=:$out/share
             '';
         });
 

misc/bash/completion.sh

@@ -7,13 +7,15 @@ function _complete_nix {
         local completion=${line%%	*}
         if [[ -z $have_type ]]; then
             have_type=1
-            if [[ $completion = filenames ]]; then
+            if [[ $completion == filenames ]]; then
                 compopt -o filenames
+            elif [[ $completion == attrs ]]; then
+                compopt -o nospace
             fi
         else
             COMPREPLY+=("$completion")
         fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}")
     __ltrim_colon_completions "$cur"
 }
 

misc/fish/completion.fish

@@ -19,7 +19,6 @@ end
 
 function _nix_accepts_files
   set -l response (_nix_complete)
-  # First line is either filenames or no-filenames.
   test $response[1] = 'filenames'
 end
 

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -25,5 +25,10 @@
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>
     <string>/dev/null</string>
+    <key>SoftResourceLimits</key>
+    <dict>
+      <key>NumberOfFiles</key>
+      <integer>4096</integer>
+    </dict>
   </dict>
 </plist>

perl/configure.ac

@@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};')
 AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).

scripts/check-hydra-status.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+# set -x
+
+
+# mapfile BUILDS_FOR_LATEST_EVAL < <(
+# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+#   jq -r '.evals[0].builds[] | @sh')
+BUILDS_FOR_LATEST_EVAL=$(
+curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+  jq -r '.evals[0].builds[]')
+
+someBuildFailed=0
+
+for buildId in $BUILDS_FOR_LATEST_EVAL; do
+  buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
+
+  buildStatus=$(echo "$buildInfo" | \
+    jq -r '.buildstatus')
+
+  if [[ "$buildStatus" -ne 0 ]]; then
+    someBuildFailed=1
+    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra"
+  fi
+done
+
+exit "$someBuildFailed"

scripts/create-darwin-volume.sh

@@ -440,7 +440,22 @@ add_nix_vol_fstab_line() {
     # shellcheck disable=SC1003,SC2026
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
-    EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF
+
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
+    # (see https://github.com/NixOS/nix/issues/5468)
+    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    # the EDITOR env.
+    #
+    # TODO: at some point we should switch to `--clean`, but it wasn't added
+    # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS
+    # minver 10.12.6 seems to have released with vim 7.4
+    cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
+#!/bin/sh
+/usr/bin/ex --noplugin "\$@"
+EOF
+    chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
+
+    EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
 UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
 .
@@ -631,7 +646,7 @@ EOF
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex /etc/synthetic.conf <<EOF
+            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -794,7 +809,7 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

scripts/install-darwin-multi-user.sh

@@ -218,7 +218,7 @@ EOF
         setup_darwin_volume
     fi
 
-    if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+    if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
-        failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix."
+        failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix."
     fi
 }

scripts/install-multi-user.sh

@@ -377,6 +377,11 @@ cure_artifacts() {
 }
 
 validate_starting_assumptions() {
+    task "Checking for artifacts of previous installs"
+    cat <<EOF
+Before I try to install, I'll check for signs Nix already is or has
+been installed on this system.
+EOF
     if type nix-env 2> /dev/null >&2; then
         warning <<EOF
 Nix already appears to be installed. This installer may run into issues.
@@ -386,6 +391,11 @@ $(uninstall_directions)
 EOF
     fi
 
+    # TODO: I think it would be good for this step to accumulate more
+    #       knowledge of older obsolete artifacts, if there are any.
+    #       We could issue a "reminder" here that the user might want
+    #       to clean them up?
+
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         # TODO: I think it would be good to accumulate a list of all
         #       of the copies so that people don't hit this 2 or 3x in

scripts/install-nix-from-closure.sh

@@ -38,7 +38,7 @@ fi
 
 # Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Linux" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+    echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2
 fi
 
 case "$(uname -s)" in
@@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do
                 echo "              providing multi-user support and better isolation for local builds."
                 echo "              Both for security and reproducibility, this method is recommended if"
                 echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo "              See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation"
                 echo ""
                 echo " --no-daemon: Simple, single-user installation that does not require root and is"
                 echo "              trivial to uninstall."
@@ -144,7 +144,7 @@ if ! [ -e "$dest" ]; then
 fi
 
 if ! [ -w "$dest" ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

src/cpptoml/LICENSE

@@ -1,18 +0,0 @@
-Copyright (c) 2014 Chase Geigle
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/libcmd/command.cc

@@ -96,6 +96,18 @@ ref<Store> EvalCommand::getEvalStore()
     return ref<Store>(evalStore);
 }
 
+ref<EvalState> EvalCommand::getEvalState()
+{
+    if (!evalState) evalState =
+#if HAVE_BOEHMGC
+        std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+#else
+        std::make_shared<EvalState>(
+#endif
+            searchPath, getEvalStore(), getStore());
+    return ref<EvalState>(evalState);
+}
+
 BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {

src/libcmd/installables.cc

@@ -191,7 +191,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         auto sep = prefix_.rfind('.');
         std::string searchWord;
         if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep, std::string::npos);
+            searchWord = prefix_.substr(sep + 1, std::string::npos);
             prefix_ = prefix_.substr(0, sep);
         } else {
             searchWord = prefix_;
@@ -203,6 +203,8 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         Value v2;
         state->autoCallFunction(*autoArgs, v1, v2);
 
+        completionType = ctAttrs;
+
         if (v2.type() == nAttrs) {
             for (auto & i : *v2.attrs) {
                 std::string name = i.name;
@@ -232,7 +234,9 @@ void completeFlakeRefWithFragment(
        prefix. */
     try {
         auto hash = prefix.find('#');
-        if (hash != std::string::npos) {
+        if (hash == std::string::npos) {
+            completeFlakeRef(evalState->store, prefix);
+        } else {
             auto fragment = prefix.substr(hash + 1);
             auto flakeRefS = std::string(prefix.substr(0, hash));
             // FIXME: do tilde expansion.
@@ -248,6 +252,8 @@ void completeFlakeRefWithFragment(
                flake. */
             attrPathPrefixes.push_back("");
 
+            completionType = ctAttrs;
+
             for (auto & attrPathPrefixS : attrPathPrefixes) {
                 auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS);
                 auto attrPathS = attrPathPrefixS + std::string(fragment);
@@ -285,12 +291,13 @@ void completeFlakeRefWithFragment(
     } catch (Error & e) {
         warn(e.msg());
     }
-
-    completeFlakeRef(evalState->store, prefix);
 }
 
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
+    if (!settings.isExperimentalFeatureEnabled(Xp::Flakes))
+        return;
+
     if (prefix == "")
         completions->add(".");
 

src/libcmd/repl.cc

@@ -28,7 +28,6 @@ extern "C" {
 #include "common-eval-args.hh"
 #include "get-drvs.hh"
 #include "derivations.hh"
-#include "affinity.hh"
 #include "globals.hh"
 #include "command.hh"
 #include "finally.hh"
@@ -284,6 +283,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
     };
 
     setupSignals();
+    Finally resetTerminal([&]() { rl_deprep_terminal(); });
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
     restoreSignals();
@@ -361,6 +361,8 @@ StringSet NixRepl::completePrefix(string prefix)
             // Quietly ignore evaluation errors.
         } catch (UndefinedVarError & e) {
             // Quietly ignore undefined variable errors.
+        } catch (BadURL & e) {
+            // Quietly ignore BadURL flake-related errors.
         }
     }
 
@@ -417,25 +419,27 @@ bool NixRepl::processLine(string line)
         std::cout
              << "The following commands are available:\n"
              << "\n"
-             << "  <expr>         Evaluate and print expression\n"
+             << "  <expr>        Evaluate and print expression\n"
-             << "  <x> = <expr>   Bind expression to variable\n"
+             << "  <x> = <expr>  Bind expression to variable\n"
-             << "  :a <expr>      Add attributes from resulting set to scope\n"
+             << "  :a <expr>     Add attributes from resulting set to scope\n"
-             << "  :b <expr>      Build derivation\n"
+             << "  :b <expr>     Build derivation\n"
-             << "  :e <expr>      Open package or function in $EDITOR\n"
+             << "  :e <expr>     Open package or function in $EDITOR\n"
-             << "  :i <expr>      Build derivation, then install result into current profile\n"
+             << "  :i <expr>     Build derivation, then install result into current profile\n"
-             << "  :l <path>      Load Nix expression and add it to scope\n"
+             << "  :l <path>     Load Nix expression and add it to scope\n"
-             << "  :lf <ref>      Load Nix flake and add it to scope\n"
+             << "  :lf <ref>     Load Nix flake and add it to scope\n"
-             << "  :p <expr>      Evaluate and print expression recursively\n"
+             << "  :p <expr>     Evaluate and print expression recursively\n"
-             << "  :q             Exit nix-repl\n"
+             << "  :q            Exit nix-repl\n"
-             << "  :r             Reload all files\n"
+             << "  :r            Reload all files\n"
-             << "  :s <expr>      Build dependencies of derivation, then start nix-shell\n"
+             << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
-             << "  :t <expr>      Describe result of evaluation\n"
+             << "  :t <expr>     Describe result of evaluation\n"
-             << "  :u <expr>      Build derivation, then start nix-shell\n"
+             << "  :u <expr>     Build derivation, then start nix-shell\n"
-             << "  :doc <expr>    Show documentation of a builtin function\n"
+             << "  :doc <expr>   Show documentation of a builtin function\n"
-             << "  :d <cmd>       Debug mode commands\n"
+             << "  :log <expr>   Show logs for a derivation\n"
-             << "  :d stack       Show call stack\n"
+             << "  :st [bool]    Enable, disable or toggle showing traces for errors\n";
-             << "  :d env         Show env stack\n"
+             << "  :d <cmd>      Debug mode commands\n"
-             << "  :d error       Show current error\n";
+             << "  :d stack      Show call stack\n"
+             << "  :d env        Show env stack\n"
+             << "  :d error      Show current error\n";
     }
 
     else if (command == ":d" || command == ":debug") {
@@ -543,7 +547,7 @@ bool NixRepl::processLine(string line)
         runNix("nix-shell", {state->store->printStorePath(drvPath)});
     }
 
-    else if (command == ":b" || command == ":i" || command == ":s") {
+    else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") {
         Value v;
         evalString(arg, v);
         StorePath drvPath = getDerivationPath(v);
@@ -557,6 +561,27 @@ bool NixRepl::processLine(string line)
                 logger->cout("  %s -> %s", outputName, state->store->printStorePath(outputPath));
         } else if (command == ":i") {
             runNix("nix-env", {"-i", drvPathRaw});
+        } else if (command == ":log") {
+            settings.readOnlyMode = true;
+            Finally roModeReset([&]() {
+                settings.readOnlyMode = false;
+            });
+            auto subs = getDefaultSubstituters();
+
+            subs.push_front(state->store);
+
+            bool foundLog = false;
+            RunPager pager;
+            for (auto & sub : subs) {
+                auto log = sub->getBuildLog(drvPath);
+                if (log) {
+                    printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri());
+                    logger->writeToStdout(*log);
+                    foundLog = true;
+                    break;
+                }
+            }
+            if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw);
         } else {
             runNix("nix-shell", {drvPathRaw});
         }
@@ -594,6 +619,18 @@ bool NixRepl::processLine(string line)
             throw Error("value does not have documentation");
     }
 
+    else if (command == ":st" || command == ":show-trace") {
+        if (arg == "false" || (arg == "" && loggerSettings.showTrace)) {
+            std::cout << "not showing error traces\n";
+            loggerSettings.showTrace = false;
+        } else if (arg == "true" || (arg == "" && !loggerSettings.showTrace)) {
+            std::cout << "showing error traces\n";
+            loggerSettings.showTrace = true;
+        } else {
+            throw Error("unexpected argument '%s' to %s", arg, command);
+        };
+    }
+
     else if (command != "")
         throw Error("unknown command '%1%'", command);
 
@@ -689,8 +726,16 @@ void NixRepl::loadFiles()
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs);
-    for (auto & i : *attrs.attrs)
+    if (displ + attrs.attrs->size() >= envSize)
-        addVarToScope(i.name, *i.value);
+        throw Error("environment full; cannot add more variables");
+
+    for (auto & i : *attrs.attrs) {
+        staticEnv.vars.emplace_back(i.name, displ);
+        env->values[displ++] = i.value;
+        varNames.insert((string) i.name);
+    }
+    staticEnv.sort();
+    staticEnv.deduplicate();
     notice("Added %1% variables.", attrs.attrs->size());
 }
 
@@ -824,12 +869,12 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
         str << "[ ";
         if (maxDepth > 0)
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (auto elem : v.listItems()) {
-                if (seen.find(v.listElems()[n]) != seen.end())
+                if (seen.count(elem))
                     str << "«repeated»";
                 else
                     try {
-                        printValue(str, *v.listElems()[n], maxDepth - 1, seen);
+                        printValue(str, *elem, maxDepth - 1, seen);
                     } catch (AssertionError & e) {
                         str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
                     }

src/libexpr/eval.cc

@@ -121,8 +121,8 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
     case tList2:
     case tListN:
         str << "[ ";
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
+        for (auto v2 : v.listItems()) {
-            printValue(str, active, *v.listElems()[n]);
+            printValue(str, active, *v2);
             str << " ";
         }
         str << "]";
@@ -521,8 +521,12 @@ Path EvalState::checkSourcePath(const Path & path_)
         }
     }
 
-    if (!found)
+    if (!found) {
-        throw RestrictedPathError("access to absolute path '%1%' is forbidden in restricted mode", abspath);
+        auto modeInformation = evalSettings.pureEval
+            ? "in pure eval mode (use '--impure' to override)"
+            : "in restricted mode";
+        throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation);
+    }
 
     /* Resolve symlinks. */
     debug(format("checking access to '%s'") % abspath);
@@ -960,8 +964,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 
 Value * EvalState::allocValue()
 {
+    /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
+       GC_malloc_many returns a linked list of objects of the given size, where the first word
+       of each object is also the pointer to the next object in the list. This also means that we
+       have to explicitly clear the first word of every object we take. */
+    if (!valueAllocCache) {
+        valueAllocCache = GC_malloc_many(sizeof(Value));
+        if (!valueAllocCache) throw std::bad_alloc();
+    }
+
+    /* GC_NEXT is a convenience macro for accessing the first word of an object.
+       Take the first list item, advance the list to the next item, and clear the next pointer. */
+    void * p = valueAllocCache;
+    GC_PTR_STORE_AND_DIRTY(&valueAllocCache, GC_NEXT(p));
+    GC_NEXT(p) = nullptr;
+
     nrValues++;
-    auto v = (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) p;
     return v;
 }
 
@@ -1314,8 +1333,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (size_t n = 0; n < elems.size(); ++n)
+    for (auto [n, v2] : enumerate(v.listItems()))
-        v.listElems()[n] = elems[n]->maybeThunk(state, env);
+        const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env);
 }
 
 
@@ -1834,7 +1853,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     bool first = !forceString;
     ValueType firstType = nString;
 
-    for (auto & i : *es) {
+    for (auto & [i_pos, i] : *es) {
         Value vTmp;
         i->eval(state, env, vTmp);
 
@@ -1855,7 +1874,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else {
-                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp), env, this);
+                throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp), env, this);
             }
         } else if (firstType == nFloat) {
             if (vTmp.type() == nInt) {
@@ -1863,12 +1882,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp), env, this);
+                throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp), env, this);
         } else
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
-            s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
+            s << state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first);
 
         first = false;
     }
@@ -1925,8 +1944,8 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (size_t n = 0; n < v.listSize(); ++n)
+            for (auto v2 : v.listItems())
-                recurse(*v.listElems()[n]);
+                recurse(*v2);
         }
     };
 
@@ -2113,12 +2132,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (size_t n = 0; n < v.listSize(); ++n) {
+            for (auto [n, v2] : enumerate(v.listItems())) {
-                result += coerceToString(pos, *v.listElems()[n],
+                result += coerceToString(pos, *v2,
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
                     /* !!! not quite correct */
-                    && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0))
+                    && (!v2->isList() || v2->listSize() != 0))
                     result += " ";
             }
             return result;

src/libexpr/eval.hh

@@ -145,6 +145,9 @@ private:
     /* Cache used by prim_match(). */
     std::shared_ptr<RegexCache> regexCache;
 
+    /* Allocation cache for GC'd Value objects. */
+    void * valueAllocCache = nullptr;
+
 public:
 
     EvalState(
@@ -362,7 +365,10 @@ public:
     /* Print statistics. */
     void printStats();
 
-    void realiseContext(const PathSet & context);
+    /* Realise the given context, and return a mapping from the placeholders
+     * used to construct the associated value to their final store path
+     */
+    [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
 private:
 

src/libexpr/flake/config.cc

@@ -38,11 +38,11 @@ void ConfigFile::apply()
 
         // FIXME: Move into libutil/config.cc.
         std::string valueS;
-        if (auto s = std::get_if<std::string>(&value))
+        if (auto* s = std::get_if<std::string>(&value))
             valueS = *s;
-        else if (auto n = std::get_if<int64_t>(&value))
+        else if (auto* n = std::get_if<int64_t>(&value))
-            valueS = fmt("%d", n);
+            valueS = fmt("%d", *n);
-        else if (auto b = std::get_if<Explicit<bool>>(&value))
+        else if (auto* b = std::get_if<Explicit<bool>>(&value))
             valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
             valueS = concatStringsSep(" ", *ss); // FIXME: evil

src/libexpr/flake/flake.cc

@@ -155,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, baseDir, true);
+            input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
     }
 
     if (!input.follows && !input.ref)
@@ -194,8 +194,8 @@ static Flake getFlake(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
-    auto flakeFile = canonPath(flakeDir + "/flake.nix");
+    auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -254,11 +254,10 @@ static Flake getFlake(
             else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
             else if (setting.value->type() == nBool)
-                flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+                flake.config.settings.insert({setting.name, Explicit<bool> { state.forceBool(*setting.value, *setting.pos) }});
             else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
-                for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
+                for (auto elem : setting.value->listItems()) {
-                    auto elem = setting.value->listElems()[n];
                     if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
@@ -345,7 +344,8 @@ LockedFlake lockFlake(
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
             const LockParent & parent,
-            const Path & parentPath)>
+            const Path & parentPath,
+            bool trustLock)>
             computeLocks;
 
         computeLocks = [&](
@@ -354,7 +354,8 @@ LockedFlake lockFlake(
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
             const LockParent & parent,
-            const Path & parentPath)
+            const Path & parentPath,
+            bool trustLock)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
@@ -465,14 +466,20 @@ LockedFlake lockFlake(
                                         .isFlake = (*lockedNode)->isFlake,
                                     });
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
-                                    auto o = input.overrides.find(i.first);
+                                    if (! trustLock) {
-                                    // If the override disappeared, we have to refetch the flake,
+                                        // It is possible that the flake has changed,
-                                    // since some of the inputs may not be present in the lockfile.
+                                        // so we must confirm all the follows that are in the lockfile are also in the flake.
-                                    if (o == input.overrides.end()) {
+                                        auto overridePath(inputPath);
-                                        mustRefetch = true;
+                                        overridePath.push_back(i.first);
-                                        // There's no point populating the rest of the fake inputs,
+                                        auto o = overrides.find(overridePath);
-                                        // since we'll refetch the flake anyways.
+                                        // If the override disappeared, we have to refetch the flake,
-                                        break;
+                                        // since some of the inputs may not be present in the lockfile.
+                                        if (o == overrides.end()) {
+                                            mustRefetch = true;
+                                            // There's no point populating the rest of the fake inputs,
+                                            // since we'll refetch the flake anyways.
+                                            break;
+                                        }
                                     }
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
@@ -481,11 +488,16 @@ LockedFlake lockFlake(
                             }
                         }
 
+                        LockParent newParent {
+                            .path = inputPath,
+                            .absolute = true
+                        };
+
                         computeLocks(
                             mustRefetch
                             ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
                             : fakeInputs,
-                            childNode, inputPath, oldLock, parent, parentPath);
+                            childNode, inputPath, oldLock, newParent, parentPath, !mustRefetch);
 
                     } else {
                         /* We need to create a new lock file entry. So fetch
@@ -542,7 +554,7 @@ LockedFlake lockFlake(
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
                                     inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
-                                newParent, localPath);
+                                newParent, localPath, false);
                         }
 
                         else {
@@ -566,11 +578,11 @@ LockedFlake lockFlake(
         };
 
         // Bring in the current ref for relative path resolution if we have it
-        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true);
 
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath, false);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))

src/libexpr/flake/flakeref.cc

@@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const
 }
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     if (fragment != "")
         throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
@@ -67,7 +70,10 @@ std::optional<FlakeRef> maybeParseFlakeRef(
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
     using namespace fetchers;
 
@@ -112,46 +118,49 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                to 'baseDir'). If so, search upward to the root of the
                repo (i.e. the directory containing .git). */
 
-            path = absPath(path, baseDir, true);
+            path = absPath(path, baseDir);
 
-            if (!S_ISDIR(lstat(path).st_mode))
+            if (isFlake) {
-                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
 
-            if (!allowMissing && !pathExists(path + "/flake.nix"))
+                if (!S_ISDIR(lstat(path).st_mode))
-                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
 
-            auto flakeRoot = path;
+                if (!allowMissing && !pathExists(path + "/flake.nix"))
-            std::string subdir;
+                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
 
-            while (flakeRoot != "/") {
+                auto flakeRoot = path;
-                if (pathExists(flakeRoot + "/.git")) {
+                std::string subdir;
-                    auto base = std::string("git+file://") + flakeRoot;
 
-                    auto parsedURL = ParsedURL{
+                while (flakeRoot != "/") {
-                        .url = base, // FIXME
+                    if (pathExists(flakeRoot + "/.git")) {
-                        .base = base,
+                        auto base = std::string("git+file://") + flakeRoot;
-                        .scheme = "git+file",
-                        .authority = "",
-                        .path = flakeRoot,
-                        .query = decodeQuery(match[2]),
-                    };
 
-                    if (subdir != "") {
+                        auto parsedURL = ParsedURL{
-                        if (parsedURL.query.count("dir"))
+                            .url = base, // FIXME
-                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            .base = base,
-                        parsedURL.query.insert_or_assign("dir", subdir);
+                            .scheme = "git+file",
+                            .authority = "",
+                            .path = flakeRoot,
+                            .query = decodeQuery(match[2]),
+                        };
+
+                        if (subdir != "") {
+                            if (parsedURL.query.count("dir"))
+                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            parsedURL.query.insert_or_assign("dir", subdir);
+                        }
+
+                        if (pathExists(flakeRoot + "/.git/shallow"))
+                            parsedURL.query.insert_or_assign("shallow", "1");
+
+                        return std::make_pair(
+                            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                            fragment);
                     }
 
-                    if (pathExists(flakeRoot + "/.git/shallow"))
+                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                        parsedURL.query.insert_or_assign("shallow", "1");
+                    flakeRoot = dirOf(flakeRoot);
-
-                    return std::make_pair(
-                        FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-                        fragment);
                 }
-
-                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                flakeRoot = dirOf(flakeRoot);
             }
 
         } else {

src/libexpr/flake/flakeref.hh

@@ -62,13 +62,19 @@ struct FlakeRef
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<FlakeRef> maybeParseFlake(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});

src/libexpr/get-drvs.cc

@@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
             state->forceList(*i->value, *i->pos);
 
             /* For each output... */
-            for (unsigned int j = 0; j < i->value->listSize(); ++j) {
+            for (auto elem : i->value->listItems()) {
                 /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
+                string name = state->forceStringNoCtx(*elem, *i->pos);
                 Bindings::iterator out = attrs->find(state->symbols.create(name));
                 if (out == attrs->end()) continue; // FIXME: throw error?
                 state->forceAttrs(*out->value);
@@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
-    for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
+    for (auto elem : outTI->listItems()) {
-        if ((*i)->type() != nString) throw errMsg;
+        if (elem->type() != nString) throw errMsg;
-        auto out = outputs.find((*i)->string.s);
+        auto out = outputs.find(elem->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
     if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n)
+        for (auto elem : v.listItems())
-            if (!checkMeta(*v.listElems()[n])) return false;
+            if (!checkMeta(*elem)) return false;
         return true;
     }
     else if (v.type() == nAttrs) {
@@ -364,10 +364,10 @@ static void getDerivations(EvalState & state, Value & vIn,
     }
 
     else if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
+        for (auto [n, elem] : enumerate(v.listItems())) {
-            string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
+            string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
-            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
+            if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/nixexpr.cc

@@ -190,7 +190,7 @@ void ExprConcatStrings::show(std::ostream & str) const
     str << "(";
     for (auto & i : *es) {
         if (first) first = false; else str << " + ";
-        str << *i;
+        str << i.second;
     }
     str << ")";
 }
@@ -490,7 +490,7 @@ void ExprConcatStrings::bindVars(const std::shared_ptr<const StaticEnv> &env)
         staticenv = env;
 
     for (auto & i : *es)
-        i->bindVars(env);
+        i.second->bindVars(env);
 }
 
 void ExprPos::bindVars(const std::shared_ptr<const StaticEnv> &env)

src/libexpr/nixexpr.hh

@@ -354,8 +354,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<Expr *> * es;
+    vector<std::pair<Pos, Expr *> > * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
+    ExprConcatStrings(const Pos & pos, bool forceString, vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     Pos* getPos() { return &pos; }
     COMMON_METHODS
@@ -392,6 +392,13 @@ struct StaticEnv
             [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
     }
 
+    void deduplicate()
+    {
+        const auto last = std::unique(vars.begin(), vars.end(),
+            [] (const Vars::value_type & a, const Vars::value_type & b) { return a.first == b.first; });
+        vars.erase(last, vars.end());
+    }
+
     Vars::const_iterator find(const Symbol & name) const
     {
         Vars::value_type key(name, 0);

src/libexpr/parser.y

@@ -153,7 +153,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
-static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
+static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<std::pair<Pos, Expr *> > & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
 
@@ -163,7 +163,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     size_t minIndent = 1000000;
     size_t curIndent = 0;
-    for (auto & i : es) {
+    for (auto & [i_pos, i] : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
@@ -193,12 +193,12 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     }
 
     /* Strip spaces from each line. */
-    vector<Expr *> * es2 = new vector<Expr *>;
+    vector<std::pair<Pos, Expr *> > * es2 = new vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
-    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
+    for (vector<std::pair<Pos, Expr *> >::iterator i = es.begin(); i != es.end(); ++i, --n) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(i->second);
         if (!e) {
             atStartOfLine = false;
             curDropped = 0;
@@ -235,11 +235,11 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
                 s2 = string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(symbols.create(s2)));
+        es2->emplace_back(i->first, new ExprString(symbols.create(s2)));
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2);
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
 }
 
 
@@ -278,7 +278,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   char * path;
   char * uri;
   std::vector<nix::AttrName> * attrNames;
-  std::vector<nix::Expr *> * string_parts;
+  std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -365,7 +365,7 @@ expr_op
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
   | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
   | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
@@ -411,7 +411,7 @@ expr_simple
   }
   | path_start PATH_END { $$ = $1; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), $1);
+      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
@@ -449,13 +449,13 @@ string_parts
   ;
 
 string_parts_interpolated
-  : string_parts_interpolated STR { $$ = $1; $1->push_back($2); }
+  : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); }
+  | DOLLAR_CURLY expr '}' { $$ = new vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<Expr *>;
+      $$ = new vector<std::pair<Pos, Expr *> >;
-      $$->push_back($1);
+      $$->emplace_back(makeCurPos(@1, data), $1);
-      $$->push_back($3);
+      $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
 
@@ -474,9 +474,9 @@ path_start
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | { $$ = new vector<Expr *>; }
+  | { $$ = new vector<std::pair<Pos, Expr *> >; }
   ;
 
 binds

src/libexpr/primops.cc

@@ -35,9 +35,10 @@ namespace nix {
 InvalidPathError::InvalidPathError(const Path & path) :
     EvalError("path '%s' is not valid", path), path(path) {}
 
-void EvalState::realiseContext(const PathSet & context)
+StringMap EvalState::realiseContext(const PathSet & context)
 {
     std::vector<DerivedPath::Built> drvs;
+    StringMap res;
 
     for (auto & i : context) {
         auto [ctxS, outputName] = decodeContext(i);
@@ -46,10 +47,12 @@ void EvalState::realiseContext(const PathSet & context)
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
             drvs.push_back({ctx, {outputName}});
+        } else {
+            res.insert_or_assign(ctxS, ctxS);
         }
     }
 
-    if (drvs.empty()) return;
+    if (drvs.empty()) return {};
 
     if (!evalSettings.enableImportFromDerivation)
         throw Error(
@@ -61,19 +64,53 @@ void EvalState::realiseContext(const PathSet & context)
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
     store->buildPaths(buildReqs);
 
+    /* Get all the output paths corresponding to the placeholders we had */
+    for (auto & [drvPath, outputs] : drvs) {
+        auto outputPaths = store->queryDerivationOutputMap(drvPath);
+        for (auto & outputName : outputs) {
+            if (outputPaths.count(outputName) == 0)
+                throw Error("derivation '%s' does not have an output named '%s'",
+                        store->printStorePath(drvPath), outputName);
+            res.insert_or_assign(
+                downstreamPlaceholder(*store, drvPath, outputName),
+                store->printStorePath(outputPaths.at(outputName))
+            );
+        }
+    }
+
     /* Add the output of this derivations to the allowed
        paths. */
     if (allowedPaths) {
-        for (auto & [drvPath, outputs] : drvs) {
+        for (auto & [_placeholder, outputPath] : res) {
-            auto outputPaths = store->queryDerivationOutputMap(drvPath);
+            allowPath(store->toRealPath(outputPath));
-            for (auto & outputName : outputs) {
-                if (outputPaths.count(outputName) == 0)
-                    throw Error("derivation '%s' does not have an output named '%s'",
-                            store->printStorePath(drvPath), outputName);
-                allowPath(outputPaths.at(outputName));
-            }
         }
     }
+
+    return res;
+}
+
+struct RealisePathFlags {
+    // Whether to check whether the path is a valid absolute path
+    bool requireAbsolutePath = true;
+    // Whether to check that the path is allowed in pure eval mode
+    bool checkForPureEval = true;
+};
+
+static Path realisePath(EvalState & state, const Pos & pos, Value & v, const RealisePathFlags flags = {})
+{
+    PathSet context;
+
+    Path path = flags.requireAbsolutePath
+        ? state.coerceToPath(pos, v, context)
+        : state.coerceToString(pos, v, context, false, false);
+
+    StringMap rewrites = state.realiseContext(context);
+
+    auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context);
+
+    return flags.checkForPureEval
+        ? state.checkSourcePath(realPath)
+        : realPath;
 }
 
 /* Add and attribute to the given attribute map from the output name to
@@ -109,11 +146,9 @@ static void mkOutputString(EvalState & state, Value & v,
    argument. */
 static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v)
 {
-    PathSet context;
+    Path path;
-    Path path = state.coerceToPath(pos, vPath, context);
-
     try {
-        state.realiseContext(context);
+        path = realisePath(state, pos, vPath);
     } catch (InvalidPathError & e) {
         throw EvalError({
             .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
@@ -124,8 +159,6 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         throw;
     }
 
-    Path realPath = state.checkSourcePath(state.toRealPath(path, context));
-
     // FIXME
     auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
         if (!state.store->isStorePath(path))
@@ -177,7 +210,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
 
     else {
         if (!vScope)
-            state.evalFile(realPath, v);
+            state.evalFile(path, v);
         else {
             state.forceAttrs(*vScope);
 
@@ -195,8 +228,8 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
             // No need to call staticEnv.sort(), because
             // args[0]->attrs is already sorted.
 
-            printTalkative("evaluating file '%1%'", realPath);
+            printTalkative("evaluating file '%1%'", path);
-            Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
+            Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
 
             e->eval(state, *env, v);
         }
@@ -281,22 +314,19 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    PathSet context;
+    Path path;
-    Path path = state.coerceToPath(pos, *args[0], context);
-
     try {
-        state.realiseContext(context);
+        path = realisePath(state, pos, *args[0]);
     } catch (InvalidPathError & e) {
         throw EvalError({
-            .msg = hintfmt(
+            .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
-                "cannot import '%1%', since path '%2%' is not valid",
-                path, e.path),
             .errPos = pos
         });
+    } catch (Error & e) {
+        e.addTrace(pos, "while importing '%s'", path);
+        throw;
     }
 
-    path = state.checkSourcePath(path);
-
     string sym = state.forceStringNoCtx(*args[1], pos);
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
@@ -335,11 +365,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
     Strings commandArgs;
-    for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
+    for (unsigned int i = 1; i < args[0]->listSize(); ++i)
         commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false));
-    }
     try {
-        state.realiseContext(context);
+        auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
     } catch (InvalidPathError & e) {
         throw EvalError({
             .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
@@ -616,8 +645,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
-    for (unsigned int n = 0; n < startSet->value->listSize(); ++n)
+    for (auto elem : startSet->value->listItems())
-        workSet.push_back(startSet->value->listElems()[n]);
+        workSet.push_back(elem);
 
     /* Get the operator. */
     Bindings::iterator op = getAttr(
@@ -662,9 +691,9 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         state.forceList(call, pos);
 
         /* Add the values returned by the operator to the work set. */
-        for (unsigned int n = 0; n < call.listSize(); ++n) {
+        for (auto elem : call.listItems()) {
-            state.forceValue(*call.listElems()[n], pos);
+            state.forceValue(*elem, pos);
-            workSet.push_back(call.listElems()[n]);
+            workSet.push_back(elem);
         }
     }
 
@@ -1013,8 +1042,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                command-line arguments to the builder. */
             else if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
-                for (unsigned int n = 0; n < i->value->listSize(); ++n) {
+                for (auto elem : i->value->listItems()) {
-                    string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
+                    string s = state.coerceToString(posDrvName, *elem, context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -1044,8 +1073,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         /* Require ‘outputs’ to be a list of strings. */
                         state.forceList(*i->value, posDrvName);
                         Strings ss;
-                        for (unsigned int n = 0; n < i->value->listSize(); ++n)
+                        for (auto elem : i->value->listItems())
-                            ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
+                            ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName));
                         handleOutputs(ss);
                     }
 
@@ -1350,10 +1379,14 @@ static RegisterPrimOp primop_storePath({
 
 static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    PathSet context;
+    Path path;
-    Path path = state.coerceToPath(pos, *args[0], context);
     try {
-        state.realiseContext(context);
+        // We don’t check the path right now, because we don’t want to throw if
+        // the path isn’t allowed, but just return false
+        // (and we can’t just catch the exception here because we still want to
+        // throw if something in the evaluation of `*args[0]` tries to access an
+        // unauthorized path)
+        path = realisePath(state, pos, *args[0], { .checkForPureEval = false });
     } catch (InvalidPathError & e) {
         throw EvalError({
             .msg = hintfmt(
@@ -1427,17 +1460,16 @@ static RegisterPrimOp primop_dirOf({
 /* Return the contents of a file as a string. */
 static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    PathSet context;
+    Path path;
-    Path path = state.coerceToPath(pos, *args[0], context);
     try {
-        state.realiseContext(context);
+        path = realisePath(state, pos, *args[0]);
     } catch (InvalidPathError & e) {
         throw EvalError({
             .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
             .errPos = pos
         });
     }
-    string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
+    string s = readFile(path);
     if (s.find((char) 0) != string::npos)
         throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
     mkString(v, s.c_str());
@@ -1460,28 +1492,26 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
     SearchPath searchPath;
 
-    for (unsigned int n = 0; n < args[0]->listSize(); ++n) {
+    for (auto v2 : args[0]->listItems()) {
-        Value & v2(*args[0]->listElems()[n]);
+        state.forceAttrs(*v2, pos);
-        state.forceAttrs(v2, pos);
 
         string prefix;
-        Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix"));
+        Bindings::iterator i = v2->attrs->find(state.symbols.create("prefix"));
-        if (i != v2.attrs->end())
+        if (i != v2->attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
 
         i = getAttr(
             state,
             "findFile",
             "path",
-            v2.attrs,
+            v2->attrs,
             pos
         );
 
-        PathSet context;
+        Path path;
-        string path = state.coerceToString(pos, *i->value, context, false, false);
 
         try {
-            state.realiseContext(context);
+            path = realisePath(state, pos, *i->value, { .requireAbsolutePath = false });
         } catch (InvalidPathError & e) {
             throw EvalError({
                 .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
@@ -1514,15 +1544,14 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
             .errPos = pos
         });
 
-    PathSet context;
+    Path path;
-    Path path = state.coerceToPath(pos, *args[1], context);
     try {
-        state.realiseContext(context);
+        path = realisePath(state, pos, *args[1]);
     } catch (InvalidPathError & e) {
         throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos);
     }
 
-    mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false));
+    mkString(v, hashFile(*ht, path).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -1539,10 +1568,9 @@ static RegisterPrimOp primop_hashFile({
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    PathSet ctx;
+    Path path;
-    Path path = state.coerceToPath(pos, *args[0], ctx);
     try {
-        state.realiseContext(ctx);
+        path = realisePath(state, pos, *args[0]);
     } catch (InvalidPathError & e) {
         throw EvalError({
             .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
@@ -1550,7 +1578,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
         });
     }
 
-    DirEntries entries = readDirectory(state.checkSourcePath(path));
+    DirEntries entries = readDirectory(path);
     state.mkAttrs(v, entries.size());
 
     for (auto & ent : entries) {
@@ -1877,7 +1905,8 @@ static void addPath(
     try {
         // FIXME: handle CA derivation outputs (where path needs to
         // be rewritten to the actual output).
-        state.realiseContext(context);
+        auto rewrites = state.realiseContext(context);
+        path = state.toRealPath(rewriteStrings(path, rewrites), context);
 
         StorePathSet refs;
 
@@ -2239,9 +2268,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     /* Get the attribute names to be removed. */
     std::set<Symbol> names;
-    for (unsigned int i = 0; i < args[1]->listSize(); ++i) {
+    for (auto elem : args[1]->listItems()) {
-        state.forceStringNoCtx(*args[1]->listElems()[i], pos);
+        state.forceStringNoCtx(*elem, pos);
-        names.insert(state.symbols.create(args[1]->listElems()[i]->string.s));
+        names.insert(state.symbols.create(elem->string.s));
     }
 
     /* Copy all attributes not in that set.  Note that we don't need
@@ -2249,7 +2278,7 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
        vector. */
     state.mkAttrs(v, args[0]->attrs->size());
     for (auto & i : *args[0]->attrs) {
-        if (names.find(i.name) == names.end())
+        if (!names.count(i.name))
             v.attrs->push_back(i);
     }
 }
@@ -2283,15 +2312,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     std::set<Symbol> seen;
 
-    for (unsigned int i = 0; i < args[0]->listSize(); ++i) {
+    for (auto v2 : args[0]->listItems()) {
-        Value & v2(*args[0]->listElems()[i]);
+        state.forceAttrs(*v2, pos);
-        state.forceAttrs(v2, pos);
 
         Bindings::iterator j = getAttr(
             state,
             "listToAttrs",
             state.sName,
-            v2.attrs,
+            v2->attrs,
             pos
         );
 
@@ -2303,7 +2331,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
                 state,
                 "listToAttrs",
                 state.sValue,
-                v2.attrs,
+                v2->attrs,
                 pos
             );
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
@@ -2370,11 +2398,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     Value * res[args[1]->listSize()];
     unsigned int found = 0;
 
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto v2 : args[1]->listItems()) {
-        Value & v2(*args[1]->listElems()[n]);
+        state.forceAttrs(*v2, pos);
-        state.forceAttrs(v2, pos);
+        Bindings::iterator i = v2->attrs->find(attrName);
-        Bindings::iterator i = v2.attrs->find(attrName);
+        if (i != v2->attrs->end())
-        if (i != v2.attrs->end())
             res[found++] = i->value;
     }
 
@@ -2649,8 +2676,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value
 {
     bool res = false;
     state.forceList(*args[1], pos);
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n)
+    for (auto elem : args[1]->listItems())
-        if (state.eqValues(*args[0], *args[1]->listElems()[n])) {
+        if (state.eqValues(*args[0], *elem)) {
             res = true;
             break;
         }
@@ -2709,8 +2736,8 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     if (args[2]->listSize()) {
         Value * vCur = args[1];
 
-        for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
+        for (auto [n, elem] : enumerate(args[2]->listItems())) {
-            Value * vs []{vCur, args[2]->listElems()[n]};
+            Value * vs []{vCur, elem};
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(*args[0], 2, vs, *vCur, pos);
         }
@@ -2740,8 +2767,8 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg
     state.forceList(*args[1], pos);
 
     Value vTmp;
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto elem : args[1]->listItems()) {
-        state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos);
+        state.callFunction(*args[0], *elem, vTmp, pos);
         bool res = state.forceBool(vTmp, pos);
         if (res == any) {
             mkBool(v, any);
@@ -2932,6 +2959,56 @@ static RegisterPrimOp primop_partition({
     .fun = prim_partition,
 });
 
+static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+
+    ValueVectorMap attrs;
+
+    for (auto vElem : args[1]->listItems()) {
+        Value res;
+        state.callFunction(*args[0], *vElem, res, pos);
+        string name = state.forceStringNoCtx(res, pos);
+        Symbol sym = state.symbols.create(name);
+        auto vector = attrs.try_emplace(sym, ValueVector()).first;
+        vector->second.push_back(vElem);
+    }
+
+    state.mkAttrs(v, attrs.size());
+
+    for (auto & i : attrs) {
+        Value * list = state.allocAttr(v, i.first);
+        auto size = i.second.size();
+        state.mkList(*list, size);
+        memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size);
+    }
+}
+
+static RegisterPrimOp primop_groupBy({
+    .name = "__groupBy",
+    .args = {"f", "list"},
+    .doc = R"(
+      Groups elements of *list* together by the string returned from the
+      function *f* called on each element. It returns an attribute set
+      where each attribute value contains the elements of *list* that are
+      mapped to the same corresponding attribute name returned by *f*.
+
+      For example,
+
+      ```nix
+      builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"]
+      ```
+
+      evaluates to
+
+      ```nix
+      { b = [ "bar" "baz" ]; f = [ "foo" ]; }
+      ```
+    )",
+    .fun = prim_groupBy,
+});
+
 static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceFunction(*args[0], pos);
@@ -3470,9 +3547,9 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * *
     res.reserve((args[1]->listSize() + 32) * sep.size());
     bool first = true;
 
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto elem : args[1]->listItems()) {
         if (first) first = false; else res += sep;
-        res += state.coerceToString(pos, *args[1]->listElems()[n], context);
+        res += state.coerceToString(pos, *elem, context);
     }
 
     mkString(v, res, context);
@@ -3501,14 +3578,14 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
 
     vector<string> from;
     from.reserve(args[0]->listSize());
-    for (unsigned int n = 0; n < args[0]->listSize(); ++n)
+    for (auto elem : args[0]->listItems())
-        from.push_back(state.forceString(*args[0]->listElems()[n], pos));
+        from.push_back(state.forceString(*elem, pos));
 
     vector<std::pair<string, PathSet>> to;
     to.reserve(args[1]->listSize());
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto elem : args[1]->listItems()) {
         PathSet ctx;
-        auto s = state.forceString(*args[1]->listElems()[n], ctx, pos);
+        auto s = state.forceString(*elem, ctx, pos);
         to.push_back(std::make_pair(std::move(s), std::move(ctx)));
     }
 
@@ -3736,7 +3813,7 @@ void EvalState::createBaseEnv()
                 .fun = primOp.fun,
                 .arity = std::max(primOp.args.size(), primOp.arity),
                 .name = symbols.create(primOp.name),
-                .args = std::move(primOp.args),
+                .args = primOp.args,
                 .doc = primOp.doc,
             });
 

src/libexpr/primops/context.cc

@@ -118,9 +118,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
             auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
             state.mkList(outputsVal, info.second.outputs.size());
             size_t i = 0;
-            for (const auto & output : info.second.outputs) {
+            for (const auto & output : info.second.outputs)
                 mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
-            }
         }
         infoVal.attrs->sort();
     }
@@ -181,8 +180,8 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                     .errPos = *i.pos
                 });
             }
-            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+            for (auto elem : iter->value->listItems()) {
-                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                auto name = state.forceStringNoCtx(*elem, *iter->pos);
                 context.insert("!" + name + "!" + string(i.name));
             }
         }

src/libexpr/primops/fromTOML.cc

@@ -1,86 +1,79 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "../../cpptoml/cpptoml.h"
+#include "../../toml11/toml.hpp"
 
 namespace nix {
 
-static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val)
 {
-    using namespace cpptoml;
-
     auto toml = state.forceStringNoCtx(*args[0], pos);
 
     std::istringstream tomlStream(toml);
 
-    std::function<void(Value &, std::shared_ptr<base>)> visit;
+    std::function<void(Value &, toml::value)> visit;
 
-    visit = [&](Value & v, std::shared_ptr<base> t) {
+    visit = [&](Value & v, toml::value t) {
 
-        if (auto t2 = t->as_table()) {
+        switch(t.type())
+        {
+            case toml::value_t::table:
+                {
+                    auto table = toml::get<toml::table>(t);
 
-            size_t size = 0;
+                    size_t size = 0;
-            for (auto & i : *t2) { (void) i; size++; }
+                    for (auto & i : table) { (void) i; size++; }
 
-            state.mkAttrs(v, size);
+                    state.mkAttrs(v, size);
 
-            for (auto & i : *t2) {
+                    for(auto & elem: table) {
-                auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
 
-                if (auto i2 = i.second->as_table_array()) {
+                        auto & v2 = *state.allocAttr(v, state.symbols.create(elem.first));
-                    size_t size2 = i2->get().size();
+                        visit(v2, elem.second);
-                    state.mkList(v2, size2);
+                    }
-                    for (size_t j = 0; j < size2; ++j)
+
-                        visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
+                    v.attrs->sort();
                 }
-                else
+                break;;
-                    visit(v2, i.second);
+            case toml::value_t::array:
-            }
+                {
+                    auto array = toml::get<std::vector<toml::value>>(t);
+
+                    size_t size = array.size();
+                    state.mkList(v, size);
+                    for (size_t i = 0; i < size; ++i)
+                        visit(*(v.listElems()[i] = state.allocValue()), array[i]);
+                }
+                break;;
+            case toml::value_t::boolean:
+                mkBool(v, toml::get<bool>(t));
+                break;;
+            case toml::value_t::integer:
+                mkInt(v, toml::get<int64_t>(t));
+                break;;
+            case toml::value_t::floating:
+                mkFloat(v, toml::get<NixFloat>(t));
+                break;;
+            case toml::value_t::string:
+                mkString(v, toml::get<std::string>(t));
+                break;;
+            case toml::value_t::local_datetime:
+            case toml::value_t::offset_datetime:
+            case toml::value_t::local_date:
+            case toml::value_t::local_time:
+                // We fail since Nix doesn't have date and time types
+                throw std::runtime_error("Dates and times are not supported");
+                break;;
+            case toml::value_t::empty:
+                mkNull(v);
+                break;;
 
-            v.attrs->sort();
         }
-
-        else if (auto t2 = t->as_array()) {
-            size_t size = t2->get().size();
-
-            state.mkList(v, size);
-
-            for (size_t i = 0; i < size; ++i)
-                visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
-        }
-
-        // Handle cases like 'a = [[{ a = true }]]', which IMHO should be
-        // parsed as a array containing an array containing a table,
-        // but instead are parsed as an array containing a table array
-        // containing a table.
-        else if (auto t2 = t->as_table_array()) {
-            size_t size = t2->get().size();
-
-            state.mkList(v, size);
-
-            for (size_t j = 0; j < size; ++j)
-                visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
-        }
-
-        else if (t->is_value()) {
-            if (auto val = t->as<int64_t>())
-                mkInt(v, val->get());
-            else if (auto val = t->as<NixFloat>())
-                mkFloat(v, val->get());
-            else if (auto val = t->as<bool>())
-                mkBool(v, val->get());
-            else if (auto val = t->as<std::string>())
-                mkString(v, val->get());
-            else
-                throw EvalError("unsupported value type in TOML");
-        }
-
-        else abort();
     };
 
     try {
-        visit(v, parser(tomlStream).parse());
+        visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
-    } catch (std::runtime_error & e) {
+    } catch (std::exception & e) { // TODO: toml::syntax_error
         throw EvalError({
             .msg = hintfmt("while parsing a TOML string: %s", e.what()),
             .errPos = pos

src/libexpr/value-to-json.cc

@@ -63,9 +63,9 @@ void printValueAsJSON(EvalState & state, bool strict,
 
         case nList: {
             auto list(out.list());
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (auto elem : v.listItems()) {
                 auto placeholder(list.placeholder());
-                printValueAsJSON(state, strict, *v.listElems()[n], pos, placeholder, context);
+                printValueAsJSON(state, strict, *elem, pos, placeholder, context);
             }
             break;
         }

src/libexpr/value-to-xml.cc

@@ -122,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
         case nList: {
             XMLOpenElement _(doc, "list");
-            for (unsigned int n = 0; n < v.listSize(); ++n)
+            for (auto v2 : v.listItems())
-                printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen, pos);
+                printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos);
             break;
         }
 

src/libexpr/value.hh

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <cassert>
+
 #include "symbol-table.hh"
 
 #if HAVE_BOEHMGC
@@ -350,6 +352,34 @@ public:
     bool isTrivial() const;
 
     std::vector<std::pair<Path, std::string>> getContext();
+
+    auto listItems()
+    {
+        struct ListIterable
+        {
+            typedef Value * const * iterator;
+            iterator _begin, _end;
+            iterator begin() const { return _begin; }
+            iterator end() const { return _end; }
+        };
+        assert(isList());
+        auto begin = listElems();
+        return ListIterable { begin, begin + listSize() };
+    }
+
+    auto listItems() const
+    {
+        struct ConstListIterable
+        {
+            typedef const Value * const * iterator;
+            iterator _begin, _end;
+            iterator begin() const { return _begin; }
+            iterator end() const { return _end; }
+        };
+        assert(isList());
+        auto begin = listElems();
+        return ConstListIterable { begin, begin + listSize() };
+    }
 };
 
 
@@ -395,9 +425,11 @@ void mkPath(Value & v, const char * s);
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
+typedef std::map<Symbol, ValueVector> ValueVectorMap;
 #endif
 
 

src/libfetchers/path.cc

@@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme
             // for security, ensure that if the parent is a store path, it's inside it
             if (store->isInStore(parent)) {
                 auto storePath = store->printStorePath(store->toStorePath(parent).first);
-                if (!isInDir(absPath, storePath))
+                if (!isDirOrInDir(absPath, storePath))
                     throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
             }
         } else

src/libfetchers/tarball.cc

@@ -176,6 +176,7 @@ struct TarballInputScheme : InputScheme
 
         if (!hasSuffix(url.path, ".zip")
             && !hasSuffix(url.path, ".tar")
+            && !hasSuffix(url.path, ".tgz")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
             && !hasSuffix(url.path, ".tar.bz2")

src/libmain/progress-bar.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
+static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n)
 {
     assert(n < fields.size());
     assert(fields[n].type == Logger::Field::tString);

src/libmain/shared.cc

@@ -15,9 +15,14 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
-#include <sys/types.h>
+#ifdef __linux__
-#include <sys/socket.h>
+#include <features.h>
-#include <netdb.h>
+#endif
+#ifdef __GLIBC__
+#include <gnu/lib-names.h>
+#include <nss.h>
+#include <dlfcn.h>
+#endif
 
 #include <openssl/crypto.h>
 
@@ -121,21 +126,30 @@ static void preloadNSS() {
        been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
        load its lookup libraries in the parent before any child gets a chance to. */
     std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
+#ifdef __GLIBC__
-
+        /* On linux, glibc will run every lookup through the nss layer.
-        /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
+         * That means every lookup goes, by default, through nscd, which acts as a local
-           We need the resolution to be done locally, as nscd socket will not be accessible in the
+         * cache.
-           sandbox. */
+         * Because we run builds in a sandbox, we also remove access to nscd otherwise
-        char * previous_env = getenv("LOCALDOMAIN");
+         * lookups would leak into the sandbox.
-        setenv("LOCALDOMAIN", "invalid", 1);
+         *
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
+         * But now we have a new problem, we need to make sure the nss_dns backend that
-            if (res) freeaddrinfo(res);
+         * does the dns lookups when nscd is not available is loaded or available.
-        }
+         *
-        if (previous_env) {
+         * We can't make it available without leaking nix's environment, so instead we'll
-             setenv("LOCALDOMAIN", previous_env, 1);
+         * load the backend, and configure nss so it does not try to run dns lookups
-        } else {
+         * through nscd.
-             unsetenv("LOCALDOMAIN");
+         *
-        }
+         * This is technically only used for builtins:fetch* functions so we only care
+         * about dns.
+         *
+         * All other platforms are unaffected.
+         */
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
+            warn("unable to load nss_dns backend");
+        // FIXME: get hosts entry from nsswitch.conf.
+        __nss_configure_lookup("hosts", "files dns");
+#endif
     });
 }
 
@@ -413,7 +427,7 @@ RunPager::RunPager()
     });
 
     pid.setKillSignal(SIGINT);
-
+    stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0);
     if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
         throw SysError("dupping stdout");
 }
@@ -424,7 +438,7 @@ RunPager::~RunPager()
     try {
         if (pid != -1) {
             std::cout.flush();
-            close(STDOUT_FILENO);
+            dup2(stdout, STDOUT_FILENO);
             pid.wait();
         }
     } catch (...) {

src/libmain/shared.hh

@@ -88,6 +88,7 @@ public:
 
 private:
     Pid pid;
+    int stdout;
 };
 
 extern volatile ::sig_atomic_t blockInt;

src/libstore/build/derivation-goal.cc

@@ -17,6 +17,7 @@
 #include <regex>
 #include <queue>
 
+#include <fstream>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -464,7 +465,6 @@ void DerivationGoal::inputsRealised()
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
-            resolvedDrv = drvResolved;
 
             auto msg = fmt("Resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
@@ -475,9 +475,9 @@ void DerivationGoal::inputsRealised()
                        worker.store.printStorePath(pathResolved),
                    });
 
-            auto resolvedGoal = worker.makeDerivationGoal(
+            resolvedDrvGoal = worker.makeDerivationGoal(
                 pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
+            addWaitee(resolvedDrvGoal);
 
             state = &DerivationGoal::resolvedFinished;
             return;
@@ -655,7 +655,7 @@ void DerivationGoal::tryLocalBuild() {
     throw Error(
         "unable to build with a primary store that isn't a local store; "
         "either pass a different '--store' or enable remote builds."
-        "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+        "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 }
 
 
@@ -938,16 +938,17 @@ void DerivationGoal::buildDone()
 }
 
 void DerivationGoal::resolvedFinished() {
-    assert(resolvedDrv);
+    assert(resolvedDrvGoal);
+    auto resolvedDrv = *resolvedDrvGoal->drv;
 
-    auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
+    auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
 
     StorePathSet outputPaths;
 
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
-        realWantedOutputs = resolvedDrv->outputNames();
+        realWantedOutputs = resolvedDrv.outputNames();
 
     for (auto & wantedOutput : realWantedOutputs) {
         assert(initialOutputs.count(wantedOutput) != 0);
@@ -979,9 +980,17 @@ void DerivationGoal::resolvedFinished() {
         outputPaths
     );
 
-    // This is potentially a bit fishy in terms of error reporting. Not sure
+    auto status = [&]() {
-    // how to do it in a cleaner way
+        auto resolvedResult = resolvedDrvGoal->getResult();
-    amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
+        switch (resolvedResult.status) {
+            case BuildResult::AlreadyValid:
+                return BuildResult::ResolvesToAlreadyValid;
+            default:
+                return resolvedResult.status;
+        }
+    }();
+
+    done(status);
 }
 
 HookReply DerivationGoal::tryBuildHook()
@@ -1329,6 +1338,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
     }
 
     worker.updateProgress();
+
+    auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or("");
+    if (traceBuiltOutputsFile != "") {
+        std::fstream fs;
+        fs.open(traceBuiltOutputsFile, std::fstream::out);
+        fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl;
+    }
 }
 
 

src/libstore/build/derivation-goal.hh

@@ -50,8 +50,8 @@ struct DerivationGoal : public Goal
     /* The path of the derivation. */
     StorePath drvPath;
 
-    /* The path of the corresponding resolved derivation */
+    /* The goal for the corresponding resolved derivation */
-    std::optional<BasicDerivation> resolvedDrv;
+    std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
     /* The specific outputs that we need to build.  Empty means all of
        them. */

src/libstore/build/local-derivation-goal.cc

@@ -1779,11 +1779,14 @@ void LocalDerivationGoal::runChild()
            i686-linux build on an x86_64-linux machine. */
         struct utsname utsbuf;
         uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
+        if ((drv->platform == "i686-linux"
-            (settings.thisSystem == "x86_64-linux" ||
+                && (settings.thisSystem == "x86_64-linux"
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+                    || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
+            || drv->platform == "armv7l-linux"
+            || drv->platform == "armv6l-linux")
+        {
             if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
+                throw SysError("cannot set 32-bit personality");
         }
 
         /* Impersonate a Linux 2.6 machine to get some determinism in

src/libstore/build/worker.cc

@@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals)
                 if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "
                             "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
                 else
                    throw Error("unable to start any build; remote machines may not have "
                             "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 
             }
             assert(!awake.empty());

src/libstore/ca-specific-schema.sql

@@ -19,3 +19,8 @@ create table if not exists RealisationsRefs (
     foreign key (referrer) references Realisations(id) on delete cascade,
     foreign key (realisationReference) references Realisations(id) on delete restrict
 );
+
+-- used by QueryRealisationReferences
+create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
+-- used by cascade deletion when ValidPaths is deleted
+create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);

src/libstore/daemon.cc

@@ -4,7 +4,6 @@
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 #include "finally.hh"
-#include "affinity.hh"
 #include "archive.hh"
 #include "derivations.hh"
 #include "args.hh"
@@ -431,25 +430,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashType(hashAlgoRaw);
             }
 
-            StringSink saved;
+            auto dumpSource = sinkToSource([&](Sink & saved) {
-            TeeSource savedNARSource(from, saved);
+                if (method == FileIngestionMethod::Recursive) {
-            RetrieveRegularNARSink savedRegular { saved };
+                    /* We parse the NAR dump through into `saved` unmodified,
-
+                       so why all this extra work? We still parse the NAR so
-            if (method == FileIngestionMethod::Recursive) {
+                       that we aren't sending arbitrary data to `saved`
-                /* Get the entire NAR dump from the client and save it to
+                       unwittingly`, and we know when the NAR ends so we don't
-                  a string so that we can pass it to
+                       consume the rest of `from` and can't parse another
-                  addToStoreFromDump(). */
+                       command. (We don't trust `addToStoreFromDump` to not
-                ParseSink sink; /* null sink; just parse the NAR */
+                       eagerly consume the entire stream it's given, past the
-                parseDump(sink, savedNARSource);
+                       length of the Nar. */
-            } else
+                    TeeSource savedNARSource(from, saved);
-                parseDump(savedRegular, from);
+                    ParseSink sink; /* null sink; just parse the NAR */
-
+                    parseDump(sink, savedNARSource);
+                } else {
+                    /* Incrementally parse the NAR file, stripping the
+                       metadata, and streaming the sole file we expect into
+                       `saved`. */
+                    RetrieveRegularNARSink savedRegular { saved };
+                    parseDump(savedRegular, from);
+                    if (!savedRegular.regular) throw Error("regular file expected");
+                }
+            });
             logger->startWork();
-            if (!savedRegular.regular) throw Error("regular file expected");
+            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
-
-            // FIXME: try to stream directly from `from`.
-            StringSource dumpSource { *saved.s };
-            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);
@@ -951,12 +955,12 @@ void processConnection(
 
     Finally finally([&]() {
         _isInterrupted = false;
-        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
+        printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
     });
 
     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
-        auto affinity = readInt(from);
+        // Obsolete CPU affinity.
-        setAffinityTo(affinity);
+        readInt(from);
     }
 
     readInt(from); // obsolete reserveSpace
@@ -984,6 +988,8 @@ void processConnection(
                 break;
             }
 
+            printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op);
+
             opCount++;
 
             try {

src/libstore/filetransfer.cc

@@ -544,13 +544,7 @@ struct curlFileTransfer : public FileTransfer
             stopWorkerThread();
         });
 
-#ifdef __linux__
+        unshareFilesystem();
-        /* Cause this thread to not share any FS attributes with the main thread,
-           because this causes setns() in restoreMountNamespace() to fail.
-           Ideally, this would happen in the std::thread() constructor. */
-        if (unshare(CLONE_FS) != 0)
-            throw SysError("unsharing filesystem state in download thread");
-#endif
 
         std::map<CURL *, std::shared_ptr<TransferItem>> items;
 

src/libstore/gc.cc

@@ -126,7 +126,17 @@ void LocalStore::addTempRoot(const StorePath & path)
             auto socketPath = stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
             state->fdRootsSocket = createUnixDomainSocket();
-            nix::connect(state->fdRootsSocket.get(), socketPath);
+            try {
+                nix::connect(state->fdRootsSocket.get(), socketPath);
+            } catch (SysError & e) {
+                /* The garbage collector may have exited, so we need to
+                   restart. */
+                if (e.errNo == ECONNREFUSED) {
+                    debug("GC socket connection refused");
+                    state->fdRootsSocket.close();
+                    goto restart;
+                }
+            }
         }
 
         try {
@@ -523,6 +533,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
                 if (!fdClient) continue;
 
+                debug("GC roots server accepted new client");
+
                 /* Process the connection in a separate thread. */
                 auto fdClient_ = fdClient.get();
                 std::thread clientThread([&, fdClient = std::move(fdClient)]() {
@@ -535,6 +547,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                         }
                     });
 
+                    /* On macOS, accepted sockets inherit the
+                       non-blocking flag from the server socket, so
+                       explicitly make it blocking. */
+                    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1)
+                        abort();
+
                     while (true) {
                         try {
                             auto path = readLine(fdClient.get());
@@ -559,7 +577,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                             } else
                                 printError("received garbage instead of a root from client");
                             writeFull(fdClient.get(), "1", false);
-                        } catch (Error &) { break; }
+                        } catch (Error & e) {
+                            debug("reading GC root from client: %s", e.msg());
+                            break;
+                        }
                     }
                 });
 

src/libstore/globals.cc

@@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures()
     /* For backwards compatibility, accept some "features" that are
        used in Nixpkgs to route builds to certain machines but don't
        actually require anything special on the machines. */
-    StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"};
+    StringSet features{"nixos-test", "benchmark", "big-parallel"};
 
     #if __linux__
     if (access("/dev/kvm", R_OK | W_OK) == 0)

src/libstore/globals.hh

@@ -21,7 +21,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : BaseSetting<unsigned int>(def, name, description, aliases)
+        : BaseSetting<unsigned int>(def, true, name, description, aliases)
     {
         options->addSetting(this);
     }
@@ -38,7 +38,7 @@ struct PluginFilesSetting : public BaseSetting<Paths>
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : BaseSetting<Paths>(def, name, description, aliases)
+        : BaseSetting<Paths>(def, true, name, description, aliases)
     {
         options->addSetting(this);
     }
@@ -130,7 +130,9 @@ public:
         {"build-max-jobs"}};
 
     Setting<unsigned int> buildCores{
-        this, getDefaultCores(), "cores",
+        this,
+        getDefaultCores(),
+        "cores",
         R"(
           Sets the value of the `NIX_BUILD_CORES` environment variable in the
           invocation of builders. Builders can use this variable at their
@@ -141,7 +143,7 @@ public:
           command line switch and defaults to `1`. The value `0` means that
           the builder should use all available CPU cores in the system.
         )",
-        {"build-cores"}};
+        {"build-cores"}, false};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
@@ -583,10 +585,11 @@ public:
           platform and generate incompatible code, so you may wish to
           cross-check the results of using this option against proper
           natively-built versions of your derivations.
-        )"};
+        )", {}, false};
 
     Setting<StringSet> systemFeatures{
-        this, getDefaultSystemFeatures(),
+        this,
+        getDefaultSystemFeatures(),
         "system-features",
         R"(
           A set of system “features” supported by this machine, e.g. `kvm`.
@@ -602,7 +605,7 @@ public:
           This setting by default includes `kvm` if `/dev/kvm` is accessible,
           and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
           that are used in Nixpkgs to route builds to specific machines.
-        )"};
+        )", {}, false};
 
     Setting<Strings> substituters{
         this,
@@ -797,6 +800,15 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+
+    Setting<StringSet> ignoredAcls{
+        this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls",
+        R"(
+          A list of ACLs that should be ignored, normally Nix attempts to
+          remove all ACLs from files and directories in the Nix store, but
+          some ACLs like `security.selinux` or `system.nfs4_acl` can't be
+          removed even by root. Therefore it's best to just ignore them.
+        )"};
 #endif
 
     Setting<Strings> hashedMirrors{

src/libstore/local-store.cc

@@ -8,6 +8,7 @@
 #include "references.hh"
 #include "callback.hh"
 #include "topo-sort.hh"
+#include "finally.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -79,7 +80,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 2;
+    const int nixCASchemaVersion = 3;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -130,6 +131,17 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
             txn.commit();
         }
 
+        if (curCASchema < 3) {
+            SQLiteTxn txn(db);
+            // Apply new indices added in this schema update.
+            db.exec(R"(
+                -- used by QueryRealisationReferences
+                create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
+                -- used by cascade deletion when ValidPaths is deleted
+                create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);
+            )");
+            txn.commit();
+        }
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }
@@ -589,9 +601,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
             throw SysError("querying extended attributes of '%s'", path);
 
         for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
-            /* Ignore SELinux security labels since these cannot be
+            if (settings.ignoredAcls.get().count(eaName)) continue;
-               removed even by root. */
-            if (eaName == "security.selinux") continue;
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
                 throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
         }
@@ -1333,13 +1343,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
         dump.resize(oldSize + want);
         auto got = 0;
+        Finally cleanup([&]() {
+            dump.resize(oldSize + got);
+        });
         try {
             got = source.read(dump.data() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
         }
-        dump.resize(oldSize + got);
     }
 
     std::unique_ptr<AutoDelete> delTempDir;

src/libstore/misc.cc

@@ -7,6 +7,7 @@
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "closure.hh"
+#include "filetransfer.hh"
 
 namespace nix {
 
@@ -100,7 +101,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
     downloadSize_ = narSize_ = 0;
 
-    ThreadPool pool;
+    // FIXME: make async.
+    ThreadPool pool(fileTransferSettings.httpConnections);
 
     struct State
     {

src/libstore/names.cc

@@ -42,7 +42,7 @@ DrvName::~DrvName()
 { }
 
 
-bool DrvName::matches(DrvName & n)
+bool DrvName::matches(const DrvName & n)
 {
     if (name != "*") {
         if (!regex) {

src/libstore/names.hh

@@ -19,7 +19,7 @@ struct DrvName
     DrvName(std::string_view s);
     ~DrvName();
 
-    bool matches(DrvName & n);
+    bool matches(const DrvName & n);
 
 private:
     std::unique_ptr<Regex> regex;

src/libstore/remote-store.cc

@@ -5,7 +5,6 @@
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
-#include "affinity.hh"
 #include "globals.hh"
 #include "derivations.hh"
 #include "pool.hh"
@@ -184,11 +183,8 @@ void RemoteStore::initConnection(Connection & conn)
         conn.to << PROTOCOL_VERSION;
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
-            int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
+            // Obsolete CPU affinity.
-            if (cpu != -1)
+            conn.to << 0;
-                conn.to << 1 << cpu;
-            else
-                conn.to << 0;
         }
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
@@ -684,6 +680,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
     Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
     auto conn(getConnection());
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) {
+        warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4");
+        try {
+            callback(nullptr);
+        } catch (...) { return callback.rethrow(); }
+    }
+
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();

src/libstore/store-api.cc

@@ -355,8 +355,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
 StringSet StoreConfig::getDefaultSystemFeatures()
 {
     auto res = settings.systemFeatures.get();
+
     if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
         res.insert("ca-derivations");
+
+    if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix))
+        res.insert("recursive-nix");
+
     return res;
 }
 
@@ -1074,7 +1079,7 @@ std::map<StorePath, StorePath> copyPaths(
                     nrFailed++;
                     if (!settings.keepGoing)
                         throw e;
-                    logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
+                    printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
                     showProgress();
                     return;
                 }

src/libstore/store-api.hh

@@ -151,9 +151,33 @@ struct BuildResult
         DependencyFailed,
         LogLimitExceeded,
         NotDeterministic,
+        ResolvesToAlreadyValid,
     } status = MiscFailure;
     std::string errorMsg;
 
+    std::string toString() const {
+        auto strStatus = [&]() {
+            switch (status) {
+                case Built: return "Built";
+                case Substituted: return "Substituted";
+                case AlreadyValid: return "AlreadyValid";
+                case PermanentFailure: return "PermanentFailure";
+                case InputRejected: return "InputRejected";
+                case OutputRejected: return "OutputRejected";
+                case TransientFailure: return "TransientFailure";
+                case CachedFailure: return "CachedFailure";
+                case TimedOut: return "TimedOut";
+                case MiscFailure: return "MiscFailure";
+                case DependencyFailed: return "DependencyFailed";
+                case LogLimitExceeded: return "LogLimitExceeded";
+                case NotDeterministic: return "NotDeterministic";
+                case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
+                default: return "Unknown";
+            };
+        }();
+        return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
+    }
+
     /* How many times this build was performed. */
     unsigned int timesBuilt = 0;
 
@@ -170,7 +194,7 @@ struct BuildResult
     time_t startTime = 0, stopTime = 0;
 
     bool success() {
-        return status == Built || status == Substituted || status == AlreadyValid;
+        return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
     }
 };
 

src/libutil/abstract-setting-to-json.hh

@@ -10,6 +10,7 @@ std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
     auto obj = AbstractSetting::toJSONObject();
     obj.emplace("value", value);
     obj.emplace("defaultValue", defaultValue);
+    obj.emplace("documentDefault", documentDefault);
     return obj;
 }
 }

src/libutil/affinity.cc

@@ -1,70 +0,0 @@
-#include "types.hh"
-#include "util.hh"
-#include "affinity.hh"
-
-#if __linux__
-#include <sched.h>
-#endif
-
-namespace nix {
-
-
-#if __linux__
-static bool didSaveAffinity = false;
-static cpu_set_t savedAffinity;
-
-std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset)
-{
-    auto count = CPU_COUNT(&cset);
-    for (int i=0; i < count; ++i)
-    {
-        os << (CPU_ISSET(i,&cset) ? "1" : "0");
-    }
-
-    return os;
-}
-#endif
-
-
-void setAffinityTo(int cpu)
-{
-#if __linux__
-    if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
-    didSaveAffinity = true;
-    debug(format("locking this thread to CPU %1%") % cpu);
-    cpu_set_t newAffinity;
-    CPU_ZERO(&newAffinity);
-    CPU_SET(cpu, &newAffinity);
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
-        printError("failed to lock thread to CPU %1%", cpu);
-#endif
-}
-
-
-int lockToCurrentCPU()
-{
-#if __linux__
-    int cpu = sched_getcpu();
-    if (cpu != -1) setAffinityTo(cpu);
-    return cpu;
-#else
-    return -1;
-#endif
-}
-
-
-void restoreAffinity()
-{
-#if __linux__
-    if (!didSaveAffinity) return;
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
-    {
-        std::ostringstream oss;
-        oss << savedAffinity;
-        printError("failed to restore CPU affinity %1%", oss.str());
-    }
-#endif
-}
-
-
-}

src/libutil/affinity.hh

@@ -1,9 +0,0 @@
-#pragma once
-
-namespace nix {
-
-void setAffinityTo(int cpu);
-int lockToCurrentCPU();
-void restoreAffinity();
-
-}

src/libutil/args.cc

@@ -39,7 +39,7 @@ void Completions::add(std::string completion, std::string description)
 bool Completion::operator<(const Completion & other) const
 { return completion < other.completion || (completion == other.completion && description < other.description); }
 
-bool pathCompletions = false;
+CompletionType completionType = ctNormal;
 std::shared_ptr<Completions> completions;
 
 std::string completionMarker = "___COMPLETE___";
@@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
 
 static void _completePath(std::string_view prefix, bool onlyDirs)
 {
-    pathCompletions = true;
+    completionType = ctFilenames;
     glob_t globbuf;
     int flags = GLOB_NOESCAPE | GLOB_TILDE;
     #ifdef GLOB_ONLYDIR

src/libutil/args.hh

@@ -237,7 +237,13 @@ public:
     void add(std::string completion, std::string description = "");
 };
 extern std::shared_ptr<Completions> completions;
-extern bool pathCompletions;
+
+enum CompletionType {
+    ctNormal,
+    ctFilenames,
+    ctAttrs
+};
+extern CompletionType completionType;
 
 std::optional<std::string> needsCompletion(std::string_view s);
 

src/libutil/config.hh

@@ -232,16 +232,19 @@ protected:
 
     T value;
     const T defaultValue;
+    const bool documentDefault;
 
 public:
 
     BaseSetting(const T & def,
+        const bool documentDefault,
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
         : AbstractSetting(name, description, aliases)
         , value(def)
         , defaultValue(def)
+        , documentDefault(documentDefault)
     { }
 
     operator const T &() const { return value; }
@@ -288,8 +291,9 @@ public:
         const T & def,
         const std::string & name,
         const std::string & description,
-        const std::set<std::string> & aliases = {})
+        const std::set<std::string> & aliases = {},
-        : BaseSetting<T>(def, name, description, aliases)
+        const bool documentDefault = true)
+        : BaseSetting<T>(def, documentDefault, name, description, aliases)
     {
         options->addSetting(this);
     }
@@ -311,7 +315,7 @@ public:
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : BaseSetting<Path>(def, name, description, aliases)
+        : BaseSetting<Path>(def, true, name, description, aliases)
         , allowEmpty(allowEmpty)
     {
         options->addSetting(this);

src/libutil/error.cc

@@ -25,7 +25,7 @@ const string & BaseError::calcWhat() const
         err.name = sname();
 
         std::ostringstream oss;
-        showErrorInfo(oss, err, false);
+        showErrorInfo(oss, err, loggerSettings.showTrace);
         what_ = oss.str();
 
         return *what_;

src/libutil/logging.hh

@@ -189,13 +189,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
 /* Print a string message if the current log level is at least the specified
    level. Note that this has to be implemented as a macro to ensure that the
    arguments are evaluated lazily. */
-#define printMsg(level, args...) \
+#define printMsgUsing(loggerParam, level, args...) \
     do { \
         auto __lvl = level; \
         if (__lvl <= nix::verbosity) { \
-            logger->log(__lvl, fmt(args)); \
+            loggerParam->log(__lvl, fmt(args)); \
         } \
     } while (0)
+#define printMsg(level, args...) printMsgUsing(logger, level, args)
 
 #define printError(args...) printMsg(lvlError, args)
 #define notice(args...) printMsg(lvlNotice, args)

src/libutil/tarfile.cc

@@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         else
             archive.check(r);
 
-        archive_entry_set_pathname(entry,
+        archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // Patch hardlink path
+        const char *original_hardlink = archive_entry_hardlink(entry);
+        if (original_hardlink) {
+            archive_entry_copy_hardlink(entry,
+                (destDir + "/" + original_hardlink).c_str());
+        }
+
         archive.check(archive_read_extract(archive.archive, entry, flags));
     }
 

src/libutil/tests/config.cc

@@ -161,7 +161,7 @@ namespace nix {
         Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
         setting.assign("value");
 
-        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
+        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#");
     }
 
     TEST(Config, setSettingAlias) {

src/libutil/thread-pool.cc

@@ -1,13 +1,10 @@
 #include "thread-pool.hh"
-#include "affinity.hh"
 
 namespace nix {
 
 ThreadPool::ThreadPool(size_t _maxThreads)
     : maxThreads(_maxThreads)
 {
-    restoreAffinity(); // FIXME
-
     if (!maxThreads) {
         maxThreads = std::thread::hardware_concurrency();
         if (!maxThreads) maxThreads = 1;

src/libutil/util.cc

@@ -1,5 +1,4 @@
 #include "util.hh"
-#include "affinity.hh"
 #include "sync.hh"
 #include "finally.hh"
 #include "serialise.hh"
@@ -512,6 +511,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
     if (!fd)
         throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
     return {std::move(fd), tmpl};
 }
 
@@ -1003,7 +1003,6 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
             if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
                 throw SysError("setting death signal");
 #endif
-            restoreAffinity();
             fun();
         } catch (std::exception & e) {
             try {
@@ -1659,6 +1658,14 @@ void restoreMountNamespace()
 #endif
 }
 
+void unshareFilesystem()
+{
+#ifdef __linux__
+    if (unshare(CLONE_FS) != 0 && errno != EPERM)
+        throw SysError("unsharing filesystem state in download thread");
+#endif
+}
+
 void restoreProcessContext(bool restoreMounts)
 {
     restoreSignals();
@@ -1666,8 +1673,6 @@ void restoreProcessContext(bool restoreMounts)
         restoreMountNamespace();
     }
 
-    restoreAffinity();
-
     #if __linux__
     if (savedStackSize) {
         struct rlimit limit;

src/libutil/util.hh

@@ -11,6 +11,7 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include <atomic>
 #include <functional>
 #include <map>
 #include <sstream>
@@ -299,7 +300,7 @@ void setStackSize(size_t stackSize);
 
 
 /* Restore the original inherited Unix process context (such as signal
-   masks, stack size, CPU affinity). */
+   masks, stack size). */
 void restoreProcessContext(bool restoreMounts = true);
 
 /* Save the current mount namespace. Ignored if called more than
@@ -310,6 +311,11 @@ void saveMountNamespace();
    if saveMountNamespace() was never called. */
 void restoreMountNamespace();
 
+/* Cause this thread to not share any FS attributes with the main
+   thread, because this causes setns() in restoreMountNamespace() to
+   fail. */
+void unshareFilesystem();
+
 
 class ExecError : public Error
 {

src/nix-build/nix-build.cc

@@ -14,7 +14,6 @@
 #include "local-fs-store.hh"
 #include "globals.hh"
 #include "derivations.hh"
-#include "affinity.hh"
 #include "util.hh"
 #include "shared.hh"
 #include "path-with-outputs.hh"
@@ -359,6 +358,7 @@ static void main_nix_build(int argc, char * * argv)
            is not set, then build bashInteractive from
            <nixpkgs>. */
         auto shell = getEnv("NIX_BUILD_SHELL");
+        std::optional<StorePath> shellDrv;
 
         if (!shell) {
 
@@ -375,8 +375,7 @@ static void main_nix_build(int argc, char * * argv)
                 auto bashDrv = store->parseStorePath(drv->queryDrvPath());
                 pathsToBuild.push_back({bashDrv});
                 pathsToCopy.insert(bashDrv);
-
+                shellDrv = bashDrv;
-                shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
                 logError(e.info());
@@ -402,6 +401,11 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (shellDrv) {
+            auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value());
+            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
+        }
+
         if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             auto resolvedDrv = drv.tryResolve(*store);
             assert(resolvedDrv && "Successfully resolved the derivation");

src/nix-channel/nix-channel.cc

@@ -88,15 +88,6 @@ static void update(const StringSet & channelNames)
     for (const auto & channel : channels) {
         auto name = channel.first;
         auto url = channel.second;
-        if (!(channelNames.empty() || channelNames.count(name)))
-            continue;
-
-        // We want to download the url to a file to see if it's a tarball while also checking if we
-        // got redirected in the process, so that we can grab the various parts of a nix channel
-        // definition from a consistent location if the redirect changes mid-download.
-        auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
-        auto filename = store->toRealPath(result.storePath);
-        url = result.effectiveUrl;
 
         // If the URL contains a version number, append it to the name
         // attribute (so that "nix-env -q" on the channels profile
@@ -109,30 +100,43 @@ static void update(const StringSet & channelNames)
 
         std::string extraAttrs;
 
-        bool unpacked = false;
+        if (!(channelNames.empty() || channelNames.count(name))) {
-        if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
+            // no need to update this channel, reuse the existing store path
-            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
+            Path symlink = profile + "/" + name;
-                        "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
+            Path storepath = dirOf(readLink(symlink));
-            unpacked = true;
+            exprs.push_back("f: rec { name = \"" + cname + "\"; type = \"derivation\"; outputs = [\"out\"]; system = \"builtin\"; outPath = builtins.storePath \"" + storepath + "\"; out = { inherit outPath; };}");
-        }
+        } else {
+            // We want to download the url to a file to see if it's a tarball while also checking if we
+            // got redirected in the process, so that we can grab the various parts of a nix channel
+            // definition from a consistent location if the redirect changes mid-download.
+            auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
+            auto filename = store->toRealPath(result.storePath);
+            url = result.effectiveUrl;
 
-        if (!unpacked) {
+            bool unpacked = false;
-            // Download the channel tarball.
+            if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
-            try {
+                runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
+                            "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
-            } catch (FileTransferError & e) {
+                unpacked = true;
-                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
             }
-        }
 
-        // Regardless of where it came from, add the expression representing this channel to accumulated expression
+            if (!unpacked) {
-        exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }");
+                // Download the channel tarball.
+                try {
+                    filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
+                } catch (FileTransferError & e) {
+                    filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
+                }
+            }
+            // Regardless of where it came from, add the expression representing this channel to accumulated expression
+            exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }");
+        }
     }
 
     // Unpack the channel tarballs into the Nix store and install them
     // into the channels profile.
     std::cerr << "unpacking channels...\n";
-    Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" };
+    Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression" };
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");

src/nix-env/nix-env.cc

@@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors)
 }
 
 
+namespace {
+
+std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view prefix) {
+    constexpr std::size_t maxResults = 3;
+    std::set<std::string> result;
+    for (const auto & drvInfo : allElems) {
+        const auto drvName = DrvName { drvInfo.queryName() };
+        if (hasPrefix(drvName.name, prefix)) {
+            result.emplace(drvName.name);
+
+            if (result.size() >= maxResults) {
+                break;
+            }
+        }
+    }
+    return result;
+}
+
+struct Match
+{
+    DrvInfo drvInfo;
+    std::size_t index;
+
+    Match(DrvInfo drvInfo_, std::size_t index_)
+    : drvInfo{std::move(drvInfo_)}
+    , index{index_}
+    {}
+};
+
+/* If a selector matches multiple derivations
+   with the same name, pick the one matching the current
+   system.  If there are still multiple derivations, pick the
+   one with the highest priority.  If there are still multiple
+   derivations, pick the one with the highest version.
+   Finally, if there are still multiple derivations,
+   arbitrarily pick the first one. */
+std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches) {
+    /* Map from package names to derivations. */
+    std::map<std::string, Match> newest;
+    StringSet multiple;
+
+    for (auto & match : matches) {
+        auto & oneDrv = match.drvInfo;
+
+        const auto drvName = DrvName { oneDrv.queryName() };
+        long comparison = 1;
+
+        const auto itOther = newest.find(drvName.name);
+
+        if (itOther != newest.end()) {
+            auto & newestDrv = itOther->second.drvInfo;
+
+            comparison =
+                oneDrv.querySystem() == newestDrv.querySystem() ? 0 :
+                oneDrv.querySystem() == settings.thisSystem ? 1 :
+                newestDrv.querySystem() == settings.thisSystem ? -1 : 0;
+            if (comparison == 0)
+                comparison = comparePriorities(state, oneDrv, newestDrv);
+            if (comparison == 0)
+                comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version);
+        }
+
+        if (comparison > 0) {
+            newest.erase(drvName.name);
+            newest.emplace(drvName.name, match);
+            multiple.erase(drvName.fullName);
+        } else if (comparison == 0) {
+            multiple.insert(drvName.fullName);
+        }
+    }
+
+    matches.clear();
+    for (auto & [name, match] : newest) {
+        if (multiple.find(name) != multiple.end())
+            warn(
+                "there are multiple derivations named '%1%'; using the first one",
+                name);
+        matches.push_back(match);
+    }
+
+    return matches;
+}
+
+} // end namespace
+
 static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
     const Strings & args, bool newestOnly)
 {
@@ -232,79 +317,42 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
         selectors.emplace_back("*");
 
     DrvInfos elems;
-    set<unsigned int> done;
+    std::set<std::size_t> done;
 
-    for (auto & i : selectors) {
+    for (auto & selector : selectors) {
-        typedef list<std::pair<DrvInfo, unsigned int> > Matches;
+        std::vector<Match> matches;
-        Matches matches;
+        for (const auto & [index, drvInfo] : enumerate(allElems)) {
-        unsigned int n = 0;
+            const auto drvName = DrvName { drvInfo.queryName() };
-        for (DrvInfos::const_iterator j = allElems.begin();
+            if (selector.matches(drvName)) {
-             j != allElems.end(); ++j, ++n)
+                ++selector.hits;
-        {
+                matches.emplace_back(drvInfo, index);
-            DrvName drvName(j->queryName());
-            if (i.matches(drvName)) {
-                i.hits++;
-                matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n));
             }
         }
 
-        /* If `newestOnly', if a selector matches multiple derivations
-           with the same name, pick the one matching the current
-           system.  If there are still multiple derivations, pick the
-           one with the highest priority.  If there are still multiple
-           derivations, pick the one with the highest version.
-           Finally, if there are still multiple derivations,
-           arbitrarily pick the first one. */
         if (newestOnly) {
-
+            matches = pickNewestOnly(state, std::move(matches));
-            /* Map from package names to derivations. */
-            typedef map<string, std::pair<DrvInfo, unsigned int> > Newest;
-            Newest newest;
-            StringSet multiple;
-
-            for (auto & j : matches) {
-                DrvName drvName(j.first.queryName());
-                long d = 1;
-
-                Newest::iterator k = newest.find(drvName.name);
-
-                if (k != newest.end()) {
-                    d = j.first.querySystem() == k->second.first.querySystem() ? 0 :
-                        j.first.querySystem() == settings.thisSystem ? 1 :
-                        k->second.first.querySystem() == settings.thisSystem ? -1 : 0;
-                    if (d == 0)
-                        d = comparePriorities(state, j.first, k->second.first);
-                    if (d == 0)
-                        d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version);
-                }
-
-                if (d > 0) {
-                    newest.erase(drvName.name);
-                    newest.insert(Newest::value_type(drvName.name, j));
-                    multiple.erase(j.first.queryName());
-                } else if (d == 0) {
-                    multiple.insert(j.first.queryName());
-                }
-            }
-
-            matches.clear();
-            for (auto & j : newest) {
-                if (multiple.find(j.second.first.queryName()) != multiple.end())
-                    printInfo(
-                        "warning: there are multiple derivations named '%1%'; using the first one",
-                        j.second.first.queryName());
-                matches.push_back(j.second);
-            }
         }
 
         /* Insert only those elements in the final list that we
            haven't inserted before. */
-        for (auto & j : matches)
+        for (auto & match : matches)
-            if (done.insert(j.second).second)
+            if (done.insert(match.index).second)
-                elems.push_back(j.first);
+                elems.push_back(match.drvInfo);
-    }
 
-    checkSelectorUse(selectors);
+        if (selector.hits == 0 && selector.fullName != "*") {
+            const auto prefixHits = searchByPrefix(allElems, selector.name);
+            
+            if (prefixHits.empty()) {
+                throw Error("selector '%1%' matches no derivations", selector.fullName);    
+            } else {
+                std::string suggestionMessage = ", maybe you meant:";
+                for (const auto & drvName : prefixHits) {
+                    suggestionMessage += fmt("\n%s", drvName);
+                }
+                throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName);
+            }
+        }
+    }
 
     return elems;
 }
@@ -1149,10 +1197,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                 } else if (v->type() == nList) {
                                     attrs2["type"] = "strings";
                                     XMLOpenElement m(xml, "meta", attrs2);
-                                    for (unsigned int j = 0; j < v->listSize(); ++j) {
+                                    for (auto elem : v->listItems()) {
-                                        if (v->listElems()[j]->type() != nString) continue;
+                                        if (elem->type() != nString) continue;
                                         XMLAttrs attrs3;
-                                        attrs3["value"] = v->listElems()[j]->string.s;
+                                        attrs3["value"] = elem->string.s;
                                         xml.writeEmptyElement("string", attrs3);
                                     }
                               } else if (v->type() == nAttrs) {

src/nix/app.cc

@@ -83,11 +83,14 @@ UnresolvedApp Installable::toApp(EvalState & state)
         auto outPath = cursor->getAttr(state.sOutPath)->getString();
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
+        auto aPname = cursor->maybeGetAttr("pname");
         auto aMeta = cursor->maybeGetAttr("meta");
         auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
         auto mainProgram =
             aMainProgram
             ? aMainProgram->getString()
+            : aPname
+            ? aPname->getString()
             : DrvName(name).name;
         auto program = outPath + "/bin/" + mainProgram;
         return UnresolvedApp { App {

src/nix/develop.cc

@@ -5,7 +5,6 @@
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 #include "derivations.hh"
-#include "affinity.hh"
 #include "progress-bar.hh"
 #include "run.hh"
 

src/nix/main.cc

@@ -310,7 +310,14 @@ void mainWrapped(int argc, char * * argv)
     Finally printCompletions([&]()
     {
         if (completions) {
-            std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n");
+            switch (completionType) {
+            case ctNormal:
+                std::cout << "normal\n"; break;
+            case ctFilenames:
+                std::cout << "filenames\n"; break;
+            case ctAttrs:
+                std::cout << "attrs\n"; break;
+            }
             for (auto & s : *completions)
                 std::cout << s.completion << "\t" << s.description << "\n";
         }

src/nix/registry.md

@@ -2,7 +2,7 @@ R""(
 
 # Description
 
-`nix flake` provides subcommands for managing *flake
+`nix registry` provides subcommands for managing *flake
 registries*. Flake registries are a convenience feature that allows
 you to refer to flakes using symbolic identifiers such as `nixpkgs`,
 rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You

src/nix/repl.md

@@ -35,14 +35,17 @@ R""(
   nix-repl> emacs.drvPath
   "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv"
 
-  nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out"
+  nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out"
 
-  nix-repl> :b x
+  nix-repl> :b drv
   this derivation produced the following outputs:
     out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello
 
   nix-repl> builtins.readFile drv
   "Hello, world!\n"
+
+  nix-repl> :log drv
+  Hello, world!
   ```
 
 # Description

src/nix/run.cc

@@ -8,7 +8,6 @@
 #include "finally.hh"
 #include "fs-accessor.hh"
 #include "progress-bar.hh"
-#include "affinity.hh"
 #include "eval.hh"
 
 #if __linux__

src/nix/run.md

@@ -43,10 +43,15 @@ program specified by the app definition.
 
 If *installable* evaluates to a derivation, it will try to execute the
 program `<out>/bin/<name>`, where *out* is the primary output store
-path of the derivation and *name* is the `meta.mainProgram` attribute
+path of the derivation, and *name* is the first of the following that
-of the derivation if it exists, and otherwise the name part of the
+exists:
-value of the `name` attribute of the derivation (e.g. if `name` is set
+
-to `hello-1.10`, it will run `$out/bin/hello`).
+* The `meta.mainProgram` attribute of the derivation.
+* The `pname` attribute of the derivation.
+* The name part of the value of the `name` attribute of the derivation.
+
+For instance, if `name` is set to `hello-1.10`, `nix run` will run
+`$out/bin/hello`.
 
 # Flake output attributes
 

src/nix/shell.md

@@ -41,8 +41,8 @@ R""(
 
 # Description
 
-`nix shell` runs a command in an environment in which the `$PATH`
+`nix shell` runs a command in an environment in which the `$PATH` variable
-variable provides the specified *installables*. If not command is
+provides the specified *installables*. If no command is specified, it starts the
-specified, it starts the default shell of your user account.
+default shell of your user account specified by `$SHELL`.
 
 )""

src/nix/why-depends.cc

@@ -34,8 +34,21 @@ struct CmdWhyDepends : SourceExprCommand
 
     CmdWhyDepends()
     {
-        expectArg("package", &_package);
+        expectArgs({
-        expectArg("dependency", &_dependency);
+            .label = "package",
+            .handler = {&_package},
+            .completer = {[&](size_t, std::string_view prefix) {
+                completeInstallable(prefix);
+            }}
+        });
+
+        expectArgs({
+            .label = "dependency",
+            .handler = {&_dependency},
+            .completer = {[&](size_t, std::string_view prefix) {
+                completeInstallable(prefix);
+            }}
+        });
 
         addFlag({
             .longName = "all",

src/toml11/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Toru Niina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

src/toml11/toml.hpp

@@ -0,0 +1,46 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Toru Niina
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef TOML_FOR_MODERN_CPP
+#define TOML_FOR_MODERN_CPP
+
+#ifndef __cplusplus
+#    error "__cplusplus is not defined"
+#endif
+
+#if __cplusplus < 201103L && _MSC_VER < 1900
+#    error "toml11 requires C++11 or later."
+#endif
+
+#define TOML11_VERSION_MAJOR 3
+#define TOML11_VERSION_MINOR 7
+#define TOML11_VERSION_PATCH 0
+
+#include "toml/parser.hpp"
+#include "toml/literal.hpp"
+#include "toml/serializer.hpp"
+#include "toml/get.hpp"
+#include "toml/macros.hpp"
+
+#endif// TOML_FOR_MODERN_CPP