Merge branch 'master' into cross-jobs

.github/STALE-BOT.md

@@ -0,0 +1,35 @@
+# Stale bot information
+
+- Thanks for your contribution!
+- To remove the stale label, just leave a new comment.
+- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.)
+- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos).
+
+## Suggestions for PRs
+
+1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else.
+2. If it is unfinished but you plan to finish it, please mark it as a draft.
+3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work.
+4. To get things rolling again, rebase the PR against the target branch and address valid comments.
+5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604).
+6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions.
+
+## Suggestions for issues
+
+1. If it is resolved (either for you personally, or in general), please consider closing it.
+2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough.
+3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/).
+4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude!
+
+**Memorandum on closing issues**
+
+Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort.
+
+## Useful GitHub search queries
+
+- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+)
+- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22)
+- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+)
+- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+)
+- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+)
+- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+)

.github/stale.yml

@@ -0,0 +1,10 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+daysUntilStale: 180
+daysUntilClose: 365
+exemptLabels:
+  - "critical"
+staleLabel: "stale"
+markComment: |
+  I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)
+closeComment: |
+  I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)

.github/workflows/test.yml

@@ -8,10 +8,52 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      CACHIX_NAME: nix-ci
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v12
+    - uses: cachix/cachix-action@v8
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
     #- run: nix flake check
     - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+  installer:
+    if: github.event_name == 'push'
+    needs: tests
+    runs-on: ubuntu-latest
+    env:
+      CACHIX_NAME: nix-ci
+    outputs:
+      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
+    steps:
+    - uses: actions/checkout@v2.3.4
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v12
+    - uses: cachix/cachix-action@v8
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+    - id: prepare-installer
+      run: scripts/prepare-installer-for-github-actions
+  installer_test:
+    if: github.event_name == 'push'
+    needs: installer
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    env:
+      CACHIX_NAME: nix-ci
+    steps:
+    - uses: actions/checkout@v2.3.4
+    - uses: cachix/install-nix-action@master
+      with:
+        install_url: '${{needs.installer.outputs.installerURL}}'
+        install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve'
+    - run: nix-instantiate -E 'builtins.currentTime' --eval
+    

Makefile.config.in

@@ -9,6 +9,7 @@ CXXFLAGS = @CXXFLAGS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
+HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
@@ -16,6 +17,7 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
+LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SHELL = @bash@

configure.ac

@@ -218,6 +218,14 @@ LDFLAGS="-lz $LDFLAGS"
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
+# Look for libcpuid.
+if test "$machine_name" = "x86_64"; then
+  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
+  have_libcpuid=1
+  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+fi
+AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
+
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then

doc/manual/generate-manpage.nix

@@ -7,7 +7,10 @@ let
 
   showCommand =
     { command, def, filename }:
-    "# Name\n\n"
+    ''
+      **Warning**: This program is **experimental** and its interface is subject to change.
+    ''
+    + "# Name\n\n"
     + "`${command}` - ${def.description}\n\n"
     + "# Synopsis\n\n"
     + showSynopsis { inherit command; args = def.args; }

doc/manual/src/advanced-topics/distributed-builds.md

@@ -37,7 +37,7 @@ then you need to ensure that the `PATH` of non-interactive login shells
 contains Nix.
 
 > **Warning**
-> 
+>
 > If you are building via the Nix daemon, it is the Nix daemon user
 > account (that is, `root`) that should have SSH access to the remote
 > machine. If you can’t or don’t want to configure `root` to be able to
@@ -52,7 +52,7 @@ example, the following command allows you to build a derivation for
 ```console
 $ uname
 Linux
-    
+
 $ nix build \
   '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
@@ -103,7 +103,7 @@ default, set it to `-`.
     ```nix
     requiredSystemFeatures = [ "kvm" ];
     ```
-    
+
     will cause the build to be performed on a machine that has the `kvm`
     feature.
 
@@ -112,6 +112,10 @@ default, set it to `-`.
     features appear in the derivation’s `requiredSystemFeatures`
     attribute..
 
+8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
+    will use its regular known-hosts file. Specifically, the field is calculated
+    via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`.
+
 For example, the machine specification
 
     nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm

doc/manual/src/command-ref/nix-shell.md

@@ -232,22 +232,23 @@ terraform apply
 > in a nix-shell shebang.
 
 Finally, using the merging of multiple nix-shell shebangs the following
-Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable
 branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
-import Network.HTTP
+import Network.Curl.Download
 import Text.HTML.TagSoup
+import Data.Either
+import Data.ByteString.Char8 (unpack)
 
 -- Fetch nixos.org and print all hrefs.
 main = do
-  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+  resp <- openURI "https://nixos.org/"
-  body <- getResponseBody resp
+  let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp
-  let tags = filter (isTagOpenName "a") $ parseTags body
   let tags' = map (fromAttrib "href") tags
   mapM_ putStrLn $ filter (/= "") tags'
 ```

doc/manual/src/command-ref/opt-common.md

@@ -134,15 +134,6 @@ Most Nix commands accept the following command-line options:
     failure in obtaining the substitutes to lead to a full build from
     source (with the related consumption of resources).
 
-  - `--no-build-hook`  
-    Disables the build hook mechanism. This allows to ignore remote
-    builders if they are setup on the machine.
-    
-    It's useful in cases where the bandwidth between the client and the
-    remote builder is too low. In that case it can take more time to
-    upload the sources to the remote builder and fetch back the result
-    than to do the computation locally.
-
   - `--readonly-mode`  
     When this option is used, no attempt is made to open the Nix
     database. Most Nix operations do need database access, so those

doc/manual/src/release-notes/rl-2.4.md

@@ -0,0 +1,8 @@
+# Release 2.4 (202X-XX-XX)
+
+  - It is now an error to modify the `plugin-files` setting via a
+    command-line flag that appears after the first non-flag argument
+    to any command, including a subcommand to `nix`. For example,
+    `nix-instantiate default.nix --plugin-files ""` must now become
+    `nix-instantiate --plugin-files "" default.nix`.
+  - Plugins that add new `nix` subcommands are now actually respected.

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1602702596,
+        "lastModified": 1614309161,
-        "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=",
+        "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ad0d20345219790533ebe06571f82ed6b034db31",
+        "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93",
         "type": "github"
       },
       "original": {

flake.nix

@@ -93,7 +93,8 @@
             gmock
           ]
           ++ lib.optional stdenv.isLinux libseccomp
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
         awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
           (aws-sdk-cpp.override {
@@ -111,6 +112,40 @@
           ];
       };
 
+    installScriptFor = systems:
+      with nixpkgsFor.x86_64-linux;
+        runCommand "installer-script"
+          { buildInputs = [ nix ];
+          }
+          ''
+            mkdir -p $out/nix-support
+
+            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
+            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            tarballPath() {
+              # Remove the store prefix
+              local path=''${1#${builtins.storeDir}/}
+              # Get the path relative to the derivation root
+              local rest=''${path#*/}
+              # Get the derivation hash
+              local drvHash=''${path%%-*}
+              echo "$drvHash/$rest"
+            }
+
+            substitute ${./scripts/install.in} $out/install \
+              ${pkgs.lib.concatMapStrings
+                (system:
+                  '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                  ''
+                )
+                systems
+              } --replace '@nixVersion@' ${version}
+
+            echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+          '';
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -200,12 +235,12 @@
 
         };
 
-        lowdown = with final; stdenv.mkDerivation {
+        lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.7.9";
+          name = "lowdown-0.8.0";
 
           src = fetchurl {
-            url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz;
+            url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
-            hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ==";
+            hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
           };
 
           #src = lowdown-src;
@@ -318,40 +353,8 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript =
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
-          with nixpkgsFor.x86_64-linux;
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
-          runCommand "installer-script"
-            { buildInputs = [ nix ];
-            }
-            ''
-              mkdir -p $out/nix-support
-
-              # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-              # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
-              tarballPath() {
-                # Remove the store prefix
-                local path=''${1#${builtins.storeDir}/}
-                # Get the path relative to the derivation root
-                local rest=''${path#*/}
-                # Get the derivation hash
-                local drvHash=''${path%%-*}
-                echo "$drvHash/$rest"
-              }
-
-              substitute ${./scripts/install.in} $out/install \
-                ${pkgs.lib.concatMapStrings
-                  (system:
-                    '' \
-                    --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                    ''
-                  )
-                  [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-                } \
-                --replace '@nixVersion@' ${version}
-
-              echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-            '';
 
         # Line coverage analysis.
         coverage =

mk/jars.mk

@@ -1,36 +0,0 @@
-define build-jar
-
-  $(1)_NAME ?= $(1)
-
-  _d := $$(strip $$($(1)_DIR))
-
-  $(1)_PATH := $$(_d)/$$($(1)_NAME).jar
-
-  $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp
-
-  _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH))
-
-  $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER)
-	@rm -rf $$($(1)_TMPDIR)
-	@mkdir -p $$($(1)_TMPDIR)
-	$$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \
-	  $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \
-	  -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH"
-	@echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest
-	$$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) .
-	@rm $$($(1)_PATH).manifest
-	@rm -rf $$($(1)_TMPDIR)
-
-  $(1)_INSTALL_DIR ?= $$(jardir)
-
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar
-
-  $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644))
-
-  install: $$($(1)_INSTALL_PATH)
-
-  jars-list += $$($(1)_PATH)
-
-  clean-files += $$($(1)_PATH)
-
-endef

mk/lib.mk

@@ -31,7 +31,6 @@ libdir ?= $(prefix)/lib
 bindir ?= $(prefix)/bin
 libexecdir ?= $(prefix)/libexec
 datadir ?= $(prefix)/share
-jardir ?= $(datadir)/java
 localstatedir ?= $(prefix)/var
 sysconfdir ?= $(prefix)/etc
 mandir ?= $(prefix)/share/man
@@ -74,7 +73,6 @@ BUILD_DEBUG ?= 1
 ifeq ($(BUILD_DEBUG), 1)
   GLOBAL_CFLAGS += -g
   GLOBAL_CXXFLAGS += -g
-  GLOBAL_JAVACFLAGS += -g
 endif
 
 
@@ -84,7 +82,6 @@ include mk/clean.mk
 include mk/install.mk
 include mk/libraries.mk
 include mk/programs.mk
-include mk/jars.mk
 include mk/patterns.mk
 include mk/templates.mk
 include mk/tests.mk
@@ -102,7 +99,6 @@ $(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
 # Instantiate stuff.
 $(foreach lib, $(libraries), $(eval $(call build-library,$(lib))))
 $(foreach prog, $(programs), $(eval $(call build-program,$(prog))))
-$(foreach jar, $(jars), $(eval $(call build-jar,$(jar))))
 $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir))))
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
@@ -113,7 +109,7 @@ $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/
 
 .PHONY: default all man help
 
-all: $(programs-list) $(libs-list) $(jars-list) $(man-pages)
+all: $(programs-list) $(libs-list) $(man-pages)
 
 man: $(man-pages)
 
@@ -137,12 +133,6 @@ ifdef libs-list
 	@echo "The following libraries can be built:"
 	@echo ""
 	@for i in $(libs-list); do echo "  $$i"; done
-endif
-ifdef jars-list
-	@echo ""
-	@echo "The following JARs can be built:"
-	@echo ""
-	@for i in $(jars-list); do echo "  $$i"; done
 endif
 	@echo ""
 	@echo "The following variables control the build:"
@@ -153,4 +143,5 @@ endif
 	@echo "  CFLAGS: Flags for the C compiler"
 	@echo "  CXX ($(CXX)): C++ compiler to be used"
 	@echo "  CXXFLAGS: Flags for the C++ compiler"
+	@echo "  CPPFLAGS: C preprocessor flags, used for both CC and CXX"
 	@$(print-var-help)

mk/patterns.mk

@@ -1,11 +1,11 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
-	$(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP

mk/run_test.sh

@@ -14,7 +14,7 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
 if [ $status -eq 0 ]; then

mk/tests.mk

@@ -8,7 +8,7 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
+	@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
 
 endef
 

mk/tracing.mk

@@ -8,8 +8,6 @@ ifeq ($(V), 0)
   trace-ld      = @echo "  LD    " $@;
   trace-ar      = @echo "  AR    " $@;
   trace-install = @echo "  INST  " $@;
-  trace-javac   = @echo "  JAVAC " $@;
-  trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
   trace-test    = @echo "  TEST  " $@;
 

scripts/install.in

@@ -60,7 +60,7 @@ case "$(uname -s).$(uname -m)" in
 esac
 
 # Use this command-line option to fetch the tarballs using nar-serve or Cachix
-if "${1:---tarball-url-prefix}"; then
+if [ "${1:-}" = "--tarball-url-prefix" ]; then
     if [ -z "${2:-}" ]; then
         oops "missing argument for --tarball-url-prefix"
     fi

scripts/prepare-installer-for-github-actions

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
+installerHash=$(echo $script | cut -b12-43 -)
+
+installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
+
+echo "::set-output name=installerURL::$installerURL"

src/build-remote/build-remote.cc

@@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv)
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
 
+        /* If we ever use the common args framework, make sure to
+           remove initPlugins below and initialize settings first.
+        */
         if (argc != 2)
             throw UsageError("called without required arguments");
 
@@ -248,7 +251,7 @@ connected:
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
-        auto outputs = readStrings<PathSet>(source);
+        auto wantedOutputs = readStrings<StringSet>(source);
 
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
 
@@ -273,6 +276,7 @@ connected:
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
+        auto outputHashes = staticOutputHashes(*store, drv);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
@@ -280,16 +284,42 @@ connected:
         if (!result.success())
             throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        StorePathSet missing;
+        std::set<Realisation> missingRealisations;
-        for (auto & path : outputs)
+        StorePathSet missingPaths;
-            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+            for (auto & outputName : wantedOutputs) {
+                auto thisOutputHash = outputHashes.at(outputName);
+                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                if (!store->queryRealisation(thisOutputId)) {
+                    debug("missing output %s", outputName);
+                    assert(result.builtOutputs.count(thisOutputId));
+                    auto newRealisation = result.builtOutputs.at(thisOutputId);
+                    missingRealisations.insert(newRealisation);
+                    missingPaths.insert(newRealisation.outPath);
+                }
+            }
+        } else {
+            auto outputPaths = drv.outputsAndOptPaths(*store);
+            for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
+                assert(hopefullyOutputPath.second);
+                if (!store->isValidPath(*hopefullyOutputPath.second))
+                    missingPaths.insert(*hopefullyOutputPath.second);
+            }
+        }
 
-        if (!missing.empty()) {
+        if (!missingPaths.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
-                for (auto & i : missing)
+                for (auto & path : missingPaths)
-                    localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
+                    localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
+            copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
+        }
+        // XXX: Should be done as part of `copyPaths`
+        for (auto & realisation : missingRealisations) {
+            // Should hold, because if the feature isn't enabled the set
+            // of missing realisations should be empty
+            settings.requireExperimentalFeature("ca-derivations");
+            store->registerDrvOutput(realisation);
         }
 
         return 0;

src/libcmd/command.cc

@@ -118,10 +118,8 @@ void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
     run(store, std::move(storePaths));
 }
 
-void StorePathCommand::run(ref<Store> store)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
 {
-    auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
-
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
 

src/libcmd/command.hh

@@ -48,6 +48,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
     ref<EvalState> getEvalState();
 
     std::shared_ptr<EvalState> evalState;
+
+    ~EvalCommand();
 };
 
 struct MixFlakeOptions : virtual Args, EvalCommand
@@ -177,13 +179,13 @@ struct StorePathsCommand : public RealisedPathsCommand
 };
 
 /* A command that operates on exactly one store path. */
-struct StorePathCommand : public InstallablesCommand
+struct StorePathCommand : public StorePathsCommand
 {
-    using StoreCommand::run;
+    using StorePathsCommand::run;
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store) override;
+    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
 };
 
 /* A helper class for registering commands globally. */

src/libcmd/installables.cc

@@ -280,6 +280,12 @@ ref<EvalState> EvalCommand::getEvalState()
     return ref<EvalState>(evalState);
 }
 
+EvalCommand::~EvalCommand()
+{
+    if (evalState)
+        evalState->printStats();
+}
+
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
@@ -496,6 +502,23 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
     return s;
 }
 
+InstallableFlake::InstallableFlake(
+    SourceExprCommand * cmd,
+    ref<EvalState> state,
+    FlakeRef && flakeRef,
+    Strings && attrPaths,
+    Strings && prefixes,
+    const flake::LockFlags & lockFlags)
+    : InstallableValue(state),
+      flakeRef(flakeRef),
+      attrPaths(attrPaths),
+      prefixes(prefixes),
+      lockFlags(lockFlags)
+{
+    if (cmd && cmd->getAutoArgs(*state)->size())
+        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
     auto lockedFlake = getLockedFlake();
@@ -628,9 +651,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
-                        getEvalState(), std::move(flakeRef),
+                        this,
+                        getEvalState(),
+                        std::move(flakeRef),
                         fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
-                        getDefaultFlakeAttrPathPrefixes(), lockFlags));
+                        getDefaultFlakeAttrPathPrefixes(),
+                        lockFlags));
                 continue;
             } catch (...) {
                 ex = std::current_exception();

src/libcmd/installables.hh

@@ -104,11 +104,13 @@ struct InstallableFlake : InstallableValue
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
-    InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
+    InstallableFlake(
-        Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
+        SourceExprCommand * cmd,
-        : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
+        ref<EvalState> state,
-          prefixes(prefixes), lockFlags(lockFlags)
+        FlakeRef && flakeRef,
-    { }
+        Strings && attrPaths,
+        Strings && prefixes,
+        const flake::LockFlags & lockFlags);
 
     std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 

src/libcmd/markdown.cc

@@ -3,9 +3,7 @@
 #include "finally.hh"
 
 #include <sys/queue.h>
-extern "C" {
 #include <lowdown.h>
-}
 
 namespace nix {
 
@@ -42,7 +40,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    if (!rndr_res)
+        throw Error("allocation error while rendering Markdown");
 
     return std::string(buf->data, buf->size);
 }

src/libexpr/eval.cc

@@ -592,10 +592,8 @@ Value & EvalState::getBuiltin(const string & name)
 
 std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
-    if (v.isPrimOp() || v.isPrimOpApp()) {
+    if (v.isPrimOp()) {
         auto v2 = &v;
-        while (v2->isPrimOpApp())
-            v2 = v2->primOpApp.left;
         if (v2->primOp->doc)
             return Doc {
                 .pos = noPos,
@@ -1381,10 +1379,10 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             } else if (!i.def) {
                 throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
-nix attempted to evaluate a function as a top level expression; in this case it must have its
+Nix attempted to evaluate a function as a top level expression; in
-arguments supplied either by default values, or passed explicitly with --arg or --argstr.
+this case it must have its arguments supplied either by default
-
+values, or passed explicitly with '--arg' or '--argstr'. See
-https://nixos.org/manual/nix/stable/#ss-functions)", i.name);
+https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
 
             }
         }

src/libexpr/nixexpr.hh

@@ -17,7 +17,7 @@ MakeError(ThrownError, AssertionError);
 MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
-MakeError(MissingArgumentError, Error);
+MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 
 

src/libfetchers/fetchers.hh

@@ -145,7 +145,13 @@ DownloadFileResult downloadFile(
     bool immutable,
     const Headers & headers = {});
 
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballMeta
+{
+    time_t lastModified;
+    std::string effectiveUrl;
+};
+
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,

src/libfetchers/github.cc

@@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified));
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(lastModified)}
+                {"lastModified", uint64_t(meta.lastModified)}
             },
             tree.storePath,
             true);

src/libfetchers/tarball.cc

@@ -109,7 +109,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-std::pair<Tree, time_t> downloadTarball(
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -127,7 +127,10 @@ std::pair<Tree, time_t> downloadTarball(
     if (cached && !cached->expired)
         return {
             Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
-            getIntAttr(cached->infoAttrs, "lastModified")
+            {
+                .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")),
+                .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url),
+            },
         };
 
     auto res = downloadFile(store, url, name, immutable, headers);
@@ -152,6 +155,7 @@ std::pair<Tree, time_t> downloadTarball(
 
     Attrs infoAttrs({
         {"lastModified", uint64_t(lastModified)},
+        {"effectiveUrl", res.effectiveUrl},
         {"etag", res.etag},
     });
 
@@ -164,7 +168,10 @@ std::pair<Tree, time_t> downloadTarball(
 
     return {
         Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
-        lastModified,
+        {
+            .lastModified = lastModified,
+            .effectiveUrl = res.effectiveUrl,
+        },
     };
 }
 
@@ -223,9 +230,11 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        Input input(_input);
+        auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false);
+        input.attrs.insert_or_assign("url", meta.effectiveUrl);
         return {std::move(tree), input};
     }
 };

src/libmain/common-args.cc

@@ -79,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     hiddenCategories.insert(cat);
 }
 
+void MixCommonArgs::initialFlagsProcessed()
+{
+    initPlugins();
+    pluginsInited();
+}
+
+
 }

src/libmain/common-args.hh

@@ -7,10 +7,14 @@ namespace nix {
 //static constexpr auto commonArgsCategory = "Miscellaneous common options";
 static constexpr auto loggingCategory = "Logging-related options";
 
-struct MixCommonArgs : virtual Args
+class MixCommonArgs : public virtual Args
 {
+    void initialFlagsProcessed() override;
+public:
     string programName;
     MixCommonArgs(const string & programName);
+protected:
+    virtual void pluginsInited() {}
 };
 
 struct MixDryRun : virtual Args

src/libstore/build/derivation-goal.hh

@@ -2,7 +2,8 @@
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
 #include "goal.hh"
 
 namespace nix {
@@ -37,6 +38,7 @@ struct InitialOutputStatus {
 
 struct InitialOutput {
     bool wanted;
+    Hash outputHash;
     std::optional<InitialOutputStatus> known;
 };
 
@@ -48,6 +50,9 @@ struct DerivationGoal : public Goal
     /* The path of the derivation. */
     StorePath drvPath;
 
+    /* The path of the corresponding resolved derivation */
+    std::optional<BasicDerivation> resolvedDrv;
+
     /* The specific outputs that we need to build.  Empty means all of
        them. */
     StringSet wantedOutputs;
@@ -60,7 +65,7 @@ struct DerivationGoal : public Goal
     bool retrySubstitution;
 
     /* The derivation stored at drvPath. */
-    std::unique_ptr<BasicDerivation> drv;
+    std::unique_ptr<Derivation> drv;
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
 
@@ -75,18 +80,6 @@ struct DerivationGoal : public Goal
 
     std::map<std::string, InitialOutput> initialOutputs;
 
-    /* User selected for running the builder. */
-    std::unique_ptr<UserLock> buildUser;
-
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* The temporary directory. */
-    Path tmpDir;
-
-    /* The path of the temporary directory in the sandbox. */
-    Path tmpDirInSandbox;
-
     /* File descriptor for the log file. */
     AutoCloseFD fdLogFile;
     std::shared_ptr<BufferedSink> logFileSink, logSink;
@@ -102,79 +95,15 @@ struct DerivationGoal : public Goal
 
     std::string currentHookLine;
 
-    /* Pipe for the builder's standard output/error. */
-    Pipe builderOut;
-
-    /* Pipe for synchronising updates to the builder namespaces. */
-    Pipe userNamespaceSync;
-
-    /* The mount namespace of the builder, used to add additional
-       paths to the sandbox as a result of recursive Nix calls. */
-    AutoCloseFD sandboxMountNamespace;
-
-    /* On Linux, whether we're doing the build in its own user
-       namespace. */
-    bool usingUserNamespace = true;
-
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
 
-    /* Whether we're currently doing a chroot build. */
-    bool useChroot = false;
-
-    Path chrootRootDir;
-
-    /* RAII object to delete the chroot directory. */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
     /* The sort of derivation we are building. */
     DerivationType derivationType;
 
-    /* Whether to run the build in a private network namespace. */
-    bool privateNetwork = false;
-
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;
 
-    /* Stuff we need to pass to initChild(). */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
-    DirsInChroot dirsInChroot;
-
-    typedef map<string, string> Environment;
-    Environment env;
-
-#if __APPLE__
-    typedef string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
-    /* Hash rewriting. */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /* The outputs paths used during the build.
-
-       - Input-addressed derivations or fixed content-addressed outputs are
-         sometimes built when some of their outputs already exist, and can not
-         be hidden via sandboxing. We use temporary locations instead and
-         rewrite after the build. Otherwise the regular predetermined paths are
-         put here.
-
-       - Floating content-addressed derivations do not know their final build
-         output paths until the outputs are hashed, so random locations are
-         used, and then renamed. The randomness helps guard against hidden
-         self-references.
-     */
-    OutputPathMap scratchOutputs;
-
     /* The final output paths of the build.
 
        - For input-addressed derivations, always the precomputed paths
@@ -187,11 +116,6 @@ struct DerivationGoal : public Goal
 
     BuildMode buildMode;
 
-    /* If we're repairing without a chroot, there may be outputs that
-       are valid but corrupt.  So we redirect these outputs to
-       temporary paths. */
-    StorePathSet redirectedBadOutputs;
-
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
@@ -199,17 +123,6 @@ struct DerivationGoal : public Goal
 
     size_t nrRounds;
 
-    /* Path registration info from the previous round, if we're
-       building multiple times. Since this contains the hash, it
-       allows us to compare whether two rounds produced the same
-       result. */
-    std::map<Path, ValidPathInfo> prevInfos;
-
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
-
-    const static Path homeDir;
-
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
 
     std::unique_ptr<Activity> act;
@@ -222,39 +135,13 @@ struct DerivationGoal : public Goal
     /* The remote machine on which we're building. */
     std::string machineName;
 
-    /* The recursive Nix daemon socket. */
-    AutoCloseFD daemonSocket;
-
-    /* The daemon main thread. */
-    std::thread daemonThread;
-
-    /* The daemon worker threads. */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /* Paths that were added via recursive Nix calls. */
-    StorePathSet addedPaths;
-
-    /* Recursive Nix calls are only allowed to build or realize paths
-       in the original input closure or added via a recursive Nix call
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-
-    friend struct RestrictedStore;
-
     DerivationGoal(const StorePath & drvPath,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
     DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
         const StringSet & wantedOutputs, Worker & worker,
         BuildMode buildMode = bmNormal);
-    ~DerivationGoal();
+    virtual ~DerivationGoal();
-
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite();
 
     void timedOut(Error && ex) override;
 
@@ -276,7 +163,7 @@ struct DerivationGoal : public Goal
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
-    void tryLocalBuild();
+    virtual void tryLocalBuild();
     void buildDone();
 
     void resolvedFinished();
@@ -284,40 +171,11 @@ struct DerivationGoal : public Goal
     /* Is the build hook willing to perform the build? */
     HookReply tryBuildHook();
 
-    /* Start building a derivation. */
+    virtual int getChildStatus();
-    void startBuilder();
-
-    /* Fill in the environment for the builder. */
-    void initEnv();
-
-    /* Setup tmp dir location. */
-    void initTmpDir();
-
-    /* Write a JSON file containing the derivation attributes. */
-    void writeStructuredAttrs();
-
-    void startDaemon();
-
-    void stopDaemon();
-
-    /* Add 'path' to the set of paths that may be referenced by the
-       outputs, and make it appear in the sandbox. */
-    void addDependency(const StorePath & path);
-
-    /* Make a file owned by the builder. */
-    void chownToBuilder(const Path & path);
-
-    /* Run the builder's process. */
-    void runChild();
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    void registerOutputs();
+    virtual void registerOutputs();
-
-    /* Check that an output meets the requirements specified by the
-       'outputChecks' attribute (or the legacy
-       '{allowed,disallowed}{References,Requisites}' attributes). */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
 
     /* Open a log file and a pipe to it. */
     Path openLogFile();
@@ -325,8 +183,18 @@ struct DerivationGoal : public Goal
     /* Close the log file. */
     void closeLogFile();
 
-    /* Delete the temporary directory, if we have one. */
+    /* Close the read side of the logger pipe. */
-    void deleteTmpDir(bool force);
+    virtual void closeReadPipes();
+
+    /* Cleanup hooks for buildDone() */
+    virtual void cleanupHookFinally();
+    virtual void cleanupPreChildKill();
+    virtual void cleanupPostChildKill();
+    virtual bool cleanupDecideWhetherDiskFull();
+    virtual void cleanupPostOutputsRegisteredModeCheck();
+    virtual void cleanupPostOutputsRegisteredModeNonCheck();
+
+    virtual bool isReadDesc(int fd);
 
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
@@ -343,17 +211,7 @@ struct DerivationGoal : public Goal
     void checkPathValidity();
 
     /* Forcibly kill the child process, if any. */
-    void killChild();
+    virtual void killChild();
-
-    /* Create alternative path calculated from but distinct from the
-       input, so we can avoid overwriting outputs (or other store paths)
-       that already exist. */
-    StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
-       derivation hash. */
-    /* FIXME add option to randomize, so we can audit whether our
-       rewrites caught everything */
-    StorePath makeFallbackPath(std::string_view outputName);
 
     void repairClosure();
 
@@ -366,4 +224,6 @@ struct DerivationGoal : public Goal
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
+MakeError(NotDeterministic, BuildError);
+
 }

src/libstore/build/entry-points.cc

@@ -2,6 +2,7 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "derivation-goal.hh"
+#include "local-store.hh"
 
 namespace nix {
 
@@ -58,6 +59,26 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
         result.status = BuildResult::MiscFailure;
         result.errorMsg = e.msg();
     }
+    // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
+    // extended to return the full realisation for each output
+    auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
+    auto outputHashes = staticOutputHashes(*this, drv);
+    for (auto & [outputName, staticOutput] : staticDrvOutputs) {
+        auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
+        if (staticOutput.second)
+            result.builtOutputs.insert_or_assign(
+                    outputId,
+                    Realisation{ outputId, *staticOutput.second}
+                    );
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+            auto realisation = this->queryRealisation(outputId);
+            if (realisation)
+                result.builtOutputs.insert_or_assign(
+                        outputId,
+                        *realisation
+                        );
+        }
+    }
 
     return result;
 }

src/libstore/build/local-derivation-goal.hh

@@ -0,0 +1,199 @@
+#pragma once
+
+#include "derivation-goal.hh"
+#include "local-store.hh"
+
+namespace nix {
+
+struct LocalDerivationGoal : public DerivationGoal
+{
+    LocalStore & getLocalStore();
+
+    /* User selected for running the builder. */
+    std::unique_ptr<UserLock> buildUser;
+
+    /* The process ID of the builder. */
+    Pid pid;
+
+    /* The temporary directory. */
+    Path tmpDir;
+
+    /* The path of the temporary directory in the sandbox. */
+    Path tmpDirInSandbox;
+
+    /* Pipe for the builder's standard output/error. */
+    Pipe builderOut;
+
+    /* Pipe for synchronising updates to the builder namespaces. */
+    Pipe userNamespaceSync;
+
+    /* The mount namespace of the builder, used to add additional
+       paths to the sandbox as a result of recursive Nix calls. */
+    AutoCloseFD sandboxMountNamespace;
+
+    /* On Linux, whether we're doing the build in its own user
+       namespace. */
+    bool usingUserNamespace = true;
+
+    /* Whether we're currently doing a chroot build. */
+    bool useChroot = false;
+
+    Path chrootRootDir;
+
+    /* RAII object to delete the chroot directory. */
+    std::shared_ptr<AutoDelete> autoDelChroot;
+
+    /* Whether to run the build in a private network namespace. */
+    bool privateNetwork = false;
+
+    /* Stuff we need to pass to initChild(). */
+    struct ChrootPath {
+        Path source;
+        bool optional;
+        ChrootPath(Path source = "", bool optional = false)
+            : source(source), optional(optional)
+        { }
+    };
+    typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
+    DirsInChroot dirsInChroot;
+
+    typedef map<string, string> Environment;
+    Environment env;
+
+#if __APPLE__
+    typedef string SandboxProfile;
+    SandboxProfile additionalSandboxProfile;
+#endif
+
+    /* Hash rewriting. */
+    StringMap inputRewrites, outputRewrites;
+    typedef map<StorePath, StorePath> RedirectedOutputs;
+    RedirectedOutputs redirectedOutputs;
+
+    /* The outputs paths used during the build.
+
+       - Input-addressed derivations or fixed content-addressed outputs are
+         sometimes built when some of their outputs already exist, and can not
+         be hidden via sandboxing. We use temporary locations instead and
+         rewrite after the build. Otherwise the regular predetermined paths are
+         put here.
+
+       - Floating content-addressed derivations do not know their final build
+         output paths until the outputs are hashed, so random locations are
+         used, and then renamed. The randomness helps guard against hidden
+         self-references.
+     */
+    OutputPathMap scratchOutputs;
+
+    /* Path registration info from the previous round, if we're
+       building multiple times. Since this contains the hash, it
+       allows us to compare whether two rounds produced the same
+       result. */
+    std::map<Path, ValidPathInfo> prevInfos;
+
+    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
+
+    const static Path homeDir;
+
+    /* The recursive Nix daemon socket. */
+    AutoCloseFD daemonSocket;
+
+    /* The daemon main thread. */
+    std::thread daemonThread;
+
+    /* The daemon worker threads. */
+    std::vector<std::thread> daemonWorkerThreads;
+
+    /* Paths that were added via recursive Nix calls. */
+    StorePathSet addedPaths;
+
+    /* Recursive Nix calls are only allowed to build or realize paths
+       in the original input closure or added via a recursive Nix call
+       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+       /nix/store/<bla> is some arbitrary path in a binary cache). */
+    bool isAllowed(const StorePath & path)
+    {
+        return inputPaths.count(path) || addedPaths.count(path);
+    }
+
+    friend struct RestrictedStore;
+
+    using DerivationGoal::DerivationGoal;
+
+    virtual ~LocalDerivationGoal() override;
+
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
+    /* The additional states. */
+    void tryLocalBuild() override;
+
+    /* Start building a derivation. */
+    void startBuilder();
+
+    /* Fill in the environment for the builder. */
+    void initEnv();
+
+    /* Setup tmp dir location. */
+    void initTmpDir();
+
+    /* Write a JSON file containing the derivation attributes. */
+    void writeStructuredAttrs();
+
+    void startDaemon();
+
+    void stopDaemon();
+
+    /* Add 'path' to the set of paths that may be referenced by the
+       outputs, and make it appear in the sandbox. */
+    void addDependency(const StorePath & path);
+
+    /* Make a file owned by the builder. */
+    void chownToBuilder(const Path & path);
+
+    int getChildStatus() override;
+
+    /* Run the builder's process. */
+    void runChild();
+
+    /* Check that the derivation outputs all exist and register them
+       as valid. */
+    void registerOutputs() override;
+
+    /* Check that an output meets the requirements specified by the
+       'outputChecks' attribute (or the legacy
+       '{allowed,disallowed}{References,Requisites}' attributes). */
+    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
+    /* Close the read side of the logger pipe. */
+    void closeReadPipes() override;
+
+    /* Cleanup hooks for buildDone() */
+    void cleanupHookFinally() override;
+    void cleanupPreChildKill() override;
+    void cleanupPostChildKill() override;
+    bool cleanupDecideWhetherDiskFull() override;
+    void cleanupPostOutputsRegisteredModeCheck() override;
+    void cleanupPostOutputsRegisteredModeNonCheck() override;
+
+    bool isReadDesc(int fd) override;
+
+    /* Delete the temporary directory, if we have one. */
+    void deleteTmpDir(bool force);
+
+    /* Forcibly kill the child process, if any. */
+    void killChild() override;
+
+    /* Create alternative path calculated from but distinct from the
+       input, so we can avoid overwriting outputs (or other store paths)
+       that already exist. */
+    StorePath makeFallbackPath(const StorePath & path);
+    /* Make a path to another based on the output name along with the
+       derivation hash. */
+    /* FIXME add option to randomize, so we can audit whether our
+       rewrites caught everything */
+    StorePath makeFallbackPath(std::string_view outputName);
+};
+
+}

src/libstore/build/worker.cc

@@ -1,7 +1,7 @@
 #include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
-#include "derivation-goal.hh"
+#include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
 #include <poll.h>
@@ -59,8 +59,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+        return !dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
+            : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -68,8 +70,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
     const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
+    return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+        return !dynamic_cast<LocalStore *>(&store)
+            ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
+            : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
     });
 }
 

src/libstore/daemon.cc

@@ -575,6 +575,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) {
+            worker_proto::write(*store, to, res.builtOutputs);
+        }
         break;
     }
 

src/libstore/derivations.cc

@@ -57,6 +57,17 @@ bool derivationIsFixed(DerivationType dt) {
     assert(false);
 }
 
+bool derivationHasKnownOutputPaths(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return true;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    case DerivationType::DeferredInputAddressed: return false;
+    };
+    assert(false);
+}
+
+
 bool derivationIsImpure(DerivationType dt) {
     switch (dt) {
     case DerivationType::InputAddressed: return false;
@@ -745,7 +756,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
@@ -756,8 +767,13 @@ std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
         StringSet newOutputNames;
         for (auto & outputName : input.second) {
             auto actualPathOpt = inputDrvOutputs.at(outputName);
-            if (!actualPathOpt)
+            if (!actualPathOpt) {
+                warn("output %s of input %s missing, aborting the resolving",
+                    outputName,
+                    store.printStorePath(input.first)
+                );
                 return std::nullopt;
+            }
             auto actualPath = *actualPathOpt;
             inputRewrites.emplace(
                 downstreamPlaceholder(store, input.first, outputName),
@@ -771,34 +787,4 @@ std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
     return resolved;
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
-{
-    auto drvPath = writeDerivation(store, *this, NoRepair, false);
-    return Derivation::tryResolve(store, drvPath);
-}
-
-std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
-{
-    // This is quite dirty and leaky, but will disappear once #4340 is merged
-    static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
-
-    {
-        auto resolutions = resolutionsCache.lock();
-        auto resolvedDrvIter = resolutions->find(drvPath);
-        if (resolvedDrvIter != resolutions->end()) {
-            auto & [_, resolvedDrv] = *resolvedDrvIter;
-                return *resolvedDrv;
-        }
-    }
-
-    /* Try resolve drv and use that path instead. */
-    auto drv = store.readDerivation(drvPath);
-    auto attempt = drv.tryResolveUncached(store);
-    if (!attempt)
-        return std::nullopt;
-    /* Store in memo table. */
-    resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
-    return *attempt;
-}
-
 }

src/libstore/derivations.hh

@@ -94,6 +94,11 @@ bool derivationIsFixed(DerivationType);
    derivation is controlled separately. Never true for non-CA derivations. */
 bool derivationIsImpure(DerivationType);
 
+/* Does the derivation knows its own output paths?
+ * Only true when there's no floating-ca derivation involved in the closure.
+ */
+bool derivationHasKnownOutputPaths(DerivationType);
+
 struct BasicDerivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
@@ -138,14 +143,10 @@ struct Derivation : BasicDerivation
 
        2. Input placeholders are replaced with realized input store paths. */
     std::optional<BasicDerivation> tryResolve(Store & store);
-    static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
 
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
-
-private:
-    std::optional<BasicDerivation> tryResolveUncached(Store & store);
 };
 
 

src/libstore/globals.cc

@@ -3,6 +3,7 @@
 #include "archive.hh"
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
+#include "compute-levels.hh"
 
 #include <algorithm>
 #include <map>
@@ -133,24 +134,29 @@ StringSet Settings::getDefaultSystemFeatures()
 
 StringSet Settings::getDefaultExtraPlatforms()
 {
+    StringSet extraPlatforms;
+
     if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1())
-        return StringSet{"i686-linux"};
+        extraPlatforms.insert("i686-linux");
-#if __APPLE__
+
+#if __linux__
+    StringSet levels = computeLevels();
+    for (auto iter = levels.begin(); iter != levels.end(); ++iter)
+        extraPlatforms.insert(*iter + "-linux");
+#elif __APPLE__
     // Rosetta 2 emulation layer can run x86_64 binaries on aarch64
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
-            return StringSet{"aarch64-darwin"};
+            extraPlatforms.insert("aarch64-darwin");
         else if (std::string{SYSTEM} == "aarch64-darwin")
-            return StringSet{"x86_64-darwin"};
+            extraPlatforms.insert("x86_64-darwin");
-        else
-            return StringSet{};
     }
 #endif
-    else
+
-        return StringSet{};
+    return extraPlatforms;
 }
 
 bool Settings::isExperimentalFeatureEnabled(const std::string & name)
@@ -159,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
     return std::find(f.begin(), f.end(), name) != f.end();
 }
 
+MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
+    : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
+    , missingFeature(feature)
+    {}
+
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+        throw MissingExperimentalFeature(name);
 }
 
 bool Settings::isWSL1()
@@ -237,8 +248,17 @@ void MaxBuildJobsSetting::set(const std::string & str, bool append)
 }
 
 
+void PluginFilesSetting::set(const std::string & str, bool append)
+{
+    if (pluginsLoaded)
+        throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand");
+    BaseSetting<Paths>::set(str, append);
+}
+
+
 void initPlugins()
 {
+    assert(!settings.pluginFiles.pluginsLoaded);
     for (const auto & pluginFile : settings.pluginFiles.get()) {
         Paths pluginFiles;
         try {
@@ -264,6 +284,9 @@ void initPlugins()
        unknown settings. */
     globalConfig.reapplyUnknownSettings();
     globalConfig.warnUnknownSettings();
+
+    /* Tell the user if they try to set plugin-files after we've already loaded */
+    settings.pluginFiles.pluginsLoaded = true;
 }
 
 }

src/libstore/globals.hh

@@ -28,6 +28,32 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
     void set(const std::string & str, bool append = false) override;
 };
 
+struct PluginFilesSetting : public BaseSetting<Paths>
+{
+    bool pluginsLoaded = false;
+
+    PluginFilesSetting(Config * options,
+        const Paths & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<Paths>(def, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    void set(const std::string & str, bool append = false) override;
+};
+
+class MissingExperimentalFeature: public Error
+{
+public:
+    std::string missingFeature;
+
+    MissingExperimentalFeature(std::string feature);
+    virtual const char* sname() const override { return "MissingExperimentalFeature"; }
+};
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -615,7 +641,7 @@ public:
           is `root`.
 
           > **Warning**
-          > 
+          >
           > Adding a user to `trusted-users` is essentially equivalent to
           > giving that user root access to the system. For example, the user
           > can set `sandbox-paths` and thereby obtain read access to
@@ -705,13 +731,13 @@ public:
           The program executes with no arguments. The program's environment
           contains the following environment variables:
 
-            - `DRV_PATH`  
+            - `DRV_PATH`
               The derivation for the built paths.
 
               Example:
               `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
 
-            - `OUT_PATHS`  
+            - `OUT_PATHS`
               Output paths of the built derivation, separated by a space
               character.
 
@@ -742,7 +768,7 @@ public:
           documentation](https://ec.haxx.se/usingcurl-netrc.html).
 
           > **Note**
-          > 
+          >
           > This must be an absolute path, and `~` is not resolved. For
           > example, `~/.netrc` won't resolve to your home directory's
           > `.netrc`.
@@ -819,7 +845,7 @@ public:
     Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
         "Number of seconds between checking free disk space."};
 
-    Setting<Paths> pluginFiles{
+    PluginFilesSetting pluginFiles{
         this, {}, "plugin-files",
         R"(
           A list of plugin files to be loaded by Nix. Each of these files will
@@ -831,6 +857,9 @@ public:
           command, and RegisterSetting to add new nix config settings. See the
           constructors for those types for more details.
 
+          Warning! These APIs are inherently unstable and may change from
+          release to release.
+
           Since these files are loaded into the same address space as Nix
           itself, they must be DSOs compatible with the instance of Nix
           running at the time (i.e. compiled against the same headers, not

src/libstore/legacy-ssh-store.cc

@@ -15,6 +15,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
     using StoreConfig::StoreConfig;
     const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
     const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -59,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         , master(
             host,
             sshKey,
+            sshPublicHostKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
             compress,
@@ -258,7 +260,9 @@ public:
 
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
-
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
+            status.builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        }
         return status;
     }
 

src/libstore/local-store.cc

@@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
+    settings.requireExperimentalFeature("ca-derivations");
     auto state(_state.lock());
     retrySQLite<void>([&]() {
         state->stmts->RegisterRealisedOutput.use()
@@ -883,7 +884,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 
 
 std::map<std::string, std::optional<StorePath>>
-LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
+LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
 {
     auto path = path_;
     auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {

src/libstore/local-store.hh

@@ -127,7 +127,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
@@ -280,7 +280,7 @@ private:
 
     void createUser(const std::string & userName, uid_t userId) override;
 
-    friend struct DerivationGoal;
+    friend struct LocalDerivationGoal;
     friend struct SubstitutionGoal;
 };
 

src/libstore/local.mk

@@ -28,7 +28,7 @@ ifeq ($(OS), SunOS)
 endif
 
 ifeq ($(HAVE_SECCOMP), 1)
-	libstore_LDFLAGS += -lseccomp
+	libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
 endif
 
 libstore_CXXFLAGS += \

src/libstore/machines.cc

@@ -54,9 +54,15 @@ ref<Store> Machine::openStore() const {
     if (hasPrefix(storeUri, "ssh://")) {
         storeParams["max-connections"] = "1";
         storeParams["log-fd"] = "4";
+    }
+
+    if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
         if (sshKey != "")
             storeParams["ssh-key"] = sshKey;
+        if (sshPublicHostKey != "")
+            storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
     }
+
     {
         auto & fs = storeParams["system-features"];
         auto append = [&](auto feats) {

src/libstore/misc.cc

@@ -22,55 +22,53 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
 
     Sync<State> state_(State{0, paths_, 0});
 
-    std::function<void(const Path &)> enqueue;
+    std::function<void(const StorePath &)> enqueue;
 
     std::condition_variable done;
 
-    enqueue = [&](const Path & path) -> void {
+    enqueue = [&](const StorePath & path) -> void {
         {
             auto state(state_.lock());
             if (state->exc) return;
-            if (!state->paths.insert(parseStorePath(path)).second) return;
+            if (!state->paths.insert(path).second) return;
             state->pending++;
         }
 
-        queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
+        queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
             // FIXME: calls to isValidPath() should be async
 
             try {
                 auto info = fut.get();
 
-                auto path = parseStorePath(pathS);
-
                 if (flipDirection) {
 
                     StorePathSet referrers;
                     queryReferrers(path, referrers);
                     for (auto & ref : referrers)
                         if (ref != path)
-                            enqueue(printStorePath(ref));
+                            enqueue(ref);
 
                     if (includeOutputs)
                         for (auto & i : queryValidDerivers(path))
-                            enqueue(printStorePath(i));
+                            enqueue(i);
 
                     if (includeDerivers && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
                             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                                enqueue(printStorePath(i));
+                                enqueue(i);
 
                 } else {
 
                     for (auto & ref : info->references)
                         if (ref != path)
-                            enqueue(printStorePath(ref));
+                            enqueue(ref);
 
                     if (includeOutputs && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
-                            if (isValidPath(i)) enqueue(printStorePath(i));
+                            if (isValidPath(i)) enqueue(i);
 
                     if (includeDerivers && info->deriver && isValidPath(*info->deriver))
-                        enqueue(printStorePath(*info->deriver));
+                        enqueue(*info->deriver);
 
                 }
 
@@ -90,7 +88,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
     };
 
     for (auto & startPath : startPaths)
-        enqueue(printStorePath(startPath));
+        enqueue(startPath);
 
     {
         auto state(state_.lock());
@@ -160,13 +158,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     };
 
     auto checkOutput = [&](
-        const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
+        const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_)
     {
         if (drvState_->lock()->done) return;
 
-        auto drvPath = parseStorePath(drvPathS);
-        auto outPath = parseStorePath(outPathS);
-
         SubstitutablePathInfos infos;
         querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
 
@@ -203,7 +198,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 return;
             }
 
-            PathSet invalid;
+            StorePathSet invalid;
             /* true for regular derivations, and CA derivations for which we
                have a trust mapping for all wanted outputs. */
             auto knownOutputPaths = true;
@@ -213,7 +208,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                     break;
                 }
                 if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
-                    invalid.insert(printStorePath(*pathOpt));
+                    invalid.insert(*pathOpt);
             }
             if (knownOutputPaths && invalid.empty()) return;
 
@@ -223,7 +218,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
-                    pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
+                    pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState));
             } else
                 mustBuildDrv(path.path, *drv);
 

src/libstore/realisation.hh

@@ -33,6 +33,8 @@ struct Realisation {
     GENERATE_CMP(Realisation, me->id, me->outPath);
 };
 
+typedef std::map<DrvOutput, Realisation> DrvOutputs;
+
 struct OpaquePath {
     StorePath path;
 

src/libstore/remote-store.cc

@@ -12,6 +12,7 @@
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -49,6 +50,21 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
     out << renderContentAddress(ca);
 }
 
+Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
+{
+    std::string rawInput = readString(from);
+    return Realisation::fromJSON(
+        nlohmann::json::parse(rawInput),
+        "remote-protocol"
+    );
+}
+void write(const Store & store, Sink & out, const Realisation & realisation)
+{ out << realisation.toJSON().dump(); }
+
+DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
+{ return DrvOutput::parse(readString(from)); }
+void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+{ out << drvOutput.to_string(); }
 
 std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
 {
@@ -664,6 +680,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     unsigned int status;
     conn->from >> status >> res.errorMsg;
     res.status = (BuildResult::Status) status;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) {
+        auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        res.builtOutputs = builtOutputs;
+    }
     return res;
 }
 

src/libstore/serve-protocol.hh

@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x205
+#define SERVE_PROTOCOL_VERSION 0x206
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

src/libstore/ssh-store.cc

@@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
     using RemoteStoreConfig::RemoteStoreConfig;
 
     const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
     const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
     const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -34,6 +35,7 @@ public:
         , master(
             host,
             sshKey,
+            sshPublicHostKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
             compress)

src/libstore/ssh.cc

@@ -2,24 +2,37 @@
 
 namespace nix {
 
-SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
+SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
     : host(host)
     , fakeSSH(host == "localhost")
     , keyFile(keyFile)
+    , sshPublicHostKey(sshPublicHostKey)
     , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
 {
     if (host == "" || hasPrefix(host, "-"))
         throw Error("invalid SSH host name '%s'", host);
+
+    auto state(state_.lock());
+    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
 }
 
 void SSHMaster::addCommonSSHOpts(Strings & args)
 {
+    auto state(state_.lock());
+
     for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS").value_or("")))
         args.push_back(i);
     if (!keyFile.empty())
         args.insert(args.end(), {"-i", keyFile});
+    if (!sshPublicHostKey.empty()) {
+        Path fileName = (Path) *state->tmpDir + "/host-key";
+        auto p = host.rfind("@");
+        string thost = p != string::npos ? string(host, p + 1) : host;
+        writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
+        args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
+    }
     if (compress)
         args.push_back("-C");
 }
@@ -87,7 +100,6 @@ Path SSHMaster::startMaster()
 
     if (state->sshMaster != -1) return state->socketPath;
 
-    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
 
     state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
 

src/libstore/ssh.hh

@@ -12,6 +12,7 @@ private:
     const std::string host;
     bool fakeSSH;
     const std::string keyFile;
+    const std::string sshPublicHostKey;
     const bool useMaster;
     const bool compress;
     const int logFD;
@@ -29,7 +30,7 @@ private:
 
 public:
 
-    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
+    SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
 
     struct Connection
     {

src/libstore/store-api.cc

@@ -366,7 +366,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
-std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
 {
     std::map<std::string, std::optional<StorePath>> outputs;
     auto drv = readInvalidDerivation(path);
@@ -376,19 +376,6 @@ std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapN
     return outputs;
 }
 
-std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
-{
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        auto resolvedDrv = Derivation::tryResolve(*this, path);
-        if (resolvedDrv) {
-            auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
-            if (isValidPath(resolvedDrvPath))
-                return queryDerivationOutputMapNoResolve(resolvedDrvPath);
-        }
-    }
-    return queryDerivationOutputMapNoResolve(path);
-}
-
 OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
     auto resp = queryPartialDerivationOutputMap(path);
     OutputPathMap result;
@@ -796,6 +783,36 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
+    RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
+{
+    StorePathSet storePaths;
+    std::set<Realisation> realisations;
+    for (auto & path : paths) {
+        storePaths.insert(path.path());
+        if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+            settings.requireExperimentalFeature("ca-derivations");
+            realisations.insert(*realisation);
+        }
+    }
+    auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+    try {
+        for (auto & realisation : realisations) {
+            dstStore->registerDrvOutput(realisation);
+        }
+    } catch (MissingExperimentalFeature & e) {
+        // Don't fail if the remote doesn't support CA derivations is it might
+        // not be within our control to change that, and we might still want
+        // to at least copy the output paths.
+        if (e.missingFeature == "ca-derivations")
+            ignoreException();
+        else
+            throw;
+    }
+
+    return pathsMap;
+}
+
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
@@ -809,7 +826,6 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     for (auto & path : storePaths)
         pathsMap.insert_or_assign(path, path);
 
-    if (missing.empty()) return pathsMap;
 
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
@@ -884,21 +900,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
             nrDone++;
             showProgress();
         });
-
     return pathsMap;
 }
 
-
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
-    SubstituteFlag substitute)
-{
-    StorePathSet closure;
-    srcStore->computeFSClosure(storePaths, closure);
-    copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
-}
-
-
 std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
 {
     std::string path;

src/libstore/store-api.hh

@@ -162,6 +162,8 @@ struct BuildResult
        non-determinism.) */
     bool isNonDeterministic = false;
 
+    DrvOutputs builtOutputs;
+
     /* The start/stop times of the build (or one of the rounds, if it
        was repeated). */
     time_t startTime = 0, stopTime = 0;
@@ -415,12 +417,6 @@ public:
        `std::nullopt`.  */
     virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
 
-    /*
-     * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
-     * the derivation
-     */
-    virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
-
     /* Query the mapping outputName=>outputPath for the given derivation.
        Assume every output has a mapping and throw an exception otherwise. */
     OutputPathMap queryDerivationOutputMap(const StorePath & path);
@@ -758,15 +754,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
    that. Returns a map of what each path was copied to the dstStore
    as. */
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
+    const RealisedPath::Set &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
-
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-
+    const StorePathSet& paths,
-/* Copy the closure of the specified paths from one store to another. */
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet & storePaths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);

src/libstore/worker-protocol.hh

@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x11b
+#define PROTOCOL_VERSION 0x11c
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -86,6 +86,8 @@ namespace worker_proto {
 MAKE_WORKER_PROTO(, std::string);
 MAKE_WORKER_PROTO(, StorePath);
 MAKE_WORKER_PROTO(, ContentAddress);
+MAKE_WORKER_PROTO(, Realisation);
+MAKE_WORKER_PROTO(, DrvOutput);
 
 MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
 

src/libutil/args.cc

@@ -14,9 +14,19 @@ void Args::addFlag(Flag && flag_)
         assert(flag->handler.arity == flag->labels.size());
     assert(flag->longName != "");
     longFlags[flag->longName] = flag;
+    for (auto & alias : flag->aliases)
+        longFlags[alias] = flag;
     if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
+void Args::removeFlag(const std::string & longName)
+{
+    auto flag = longFlags.find(longName);
+    assert(flag != longFlags.end());
+    if (flag->second->shortName) shortFlags.erase(flag->second->shortName);
+    longFlags.erase(flag);
+}
+
 void Completions::add(std::string completion, std::string description)
 {
     assert(description.find('\n') == std::string::npos);
@@ -58,6 +68,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         verbosity = lvlError;
     }
 
+    bool argsSeen = false;
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
         auto arg = *pos;
@@ -86,6 +97,10 @@ void Args::parseCmdline(const Strings & _cmdline)
                 throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
+            if (!argsSeen) {
+                argsSeen = true;
+                initialFlagsProcessed();
+            }
             pos = rewriteArgs(cmdline, pos);
             pendingArgs.push_back(*pos++);
             if (processArgs(pendingArgs, false))
@@ -94,6 +109,9 @@ void Args::parseCmdline(const Strings & _cmdline)
     }
 
     processArgs(pendingArgs, true);
+
+    if (!argsSeen)
+        initialFlagsProcessed();
 }
 
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
@@ -191,6 +209,7 @@ nlohmann::json Args::toJSON()
 
     for (auto & [name, flag] : longFlags) {
         auto j = nlohmann::json::object();
+        if (flag->aliases.count(name)) continue;
         if (flag->shortName)
             j["shortName"] = std::string(1, flag->shortName);
         if (flag->description != "")
@@ -295,8 +314,8 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
-MultiCommand::MultiCommand(const Commands & commands)
+MultiCommand::MultiCommand(const Commands & commands_)
-    : commands(commands)
+    : commands(commands_)
 {
     expectArgs({
         .label = "subcommand",

src/libutil/args.hh

@@ -97,6 +97,7 @@ protected:
         typedef std::shared_ptr<Flag> ptr;
 
         std::string longName;
+        std::set<std::string> aliases;
         char shortName = 0;
         std::string description;
         std::string category;
@@ -131,10 +132,16 @@ protected:
 
     std::set<std::string> hiddenCategories;
 
+    /* Called after all command line flags before the first non-flag
+       argument (if any) have been processed. */
+    virtual void initialFlagsProcessed() {}
+
 public:
 
     void addFlag(Flag && flag);
 
+    void removeFlag(const std::string & longName);
+
     void expectArgs(ExpectedArg && arg)
     {
         expectedArgs.emplace_back(std::move(arg));

src/libutil/compute-levels.cc

@@ -0,0 +1,80 @@
+#include "types.hh"
+
+#if HAVE_LIBCPUID
+#include <libcpuid/libcpuid.h>
+#endif
+
+namespace nix {
+
+#if HAVE_LIBCPUID
+
+StringSet computeLevels() {
+    StringSet levels;
+
+    if (!cpuid_present())
+        return levels;
+
+    cpu_raw_data_t raw;
+    cpu_id_t data;
+
+    if (cpuid_get_raw_data(&raw) < 0)
+        return levels;
+
+    if (cpu_identify(&raw, &data) < 0)
+        return levels;
+
+    if (!(data.flags[CPU_FEATURE_CMOV] &&
+            data.flags[CPU_FEATURE_CX8] &&
+            data.flags[CPU_FEATURE_FPU] &&
+            data.flags[CPU_FEATURE_FXSR] &&
+            data.flags[CPU_FEATURE_MMX] &&
+            data.flags[CPU_FEATURE_SSE] &&
+            data.flags[CPU_FEATURE_SSE2]))
+        return levels;
+
+    levels.insert("x86_64-v1");
+
+    if (!(data.flags[CPU_FEATURE_CX16] &&
+            data.flags[CPU_FEATURE_LAHF_LM] &&
+            data.flags[CPU_FEATURE_POPCNT] &&
+            // SSE3
+            data.flags[CPU_FEATURE_PNI] &&
+            data.flags[CPU_FEATURE_SSSE3] &&
+            data.flags[CPU_FEATURE_SSE4_1] &&
+            data.flags[CPU_FEATURE_SSE4_2]))
+        return levels;
+
+    levels.insert("x86_64-v2");
+
+    if (!(data.flags[CPU_FEATURE_AVX] &&
+            data.flags[CPU_FEATURE_AVX2] &&
+            data.flags[CPU_FEATURE_F16C] &&
+            data.flags[CPU_FEATURE_FMA3] &&
+            // LZCNT
+            data.flags[CPU_FEATURE_ABM] &&
+            data.flags[CPU_FEATURE_MOVBE]))
+        return levels;
+
+    levels.insert("x86_64-v3");
+
+    if (!(data.flags[CPU_FEATURE_AVX512F] &&
+            data.flags[CPU_FEATURE_AVX512BW] &&
+            data.flags[CPU_FEATURE_AVX512CD] &&
+            data.flags[CPU_FEATURE_AVX512DQ] &&
+            data.flags[CPU_FEATURE_AVX512VL]))
+        return levels;
+
+    levels.insert("x86_64-v4");
+
+    return levels;
+}
+
+#else
+
+StringSet computeLevels() {
+    return StringSet{};
+}
+
+#endif // HAVE_LIBCPUID
+
+}

src/libutil/compute-levels.hh

@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+StringSet computeLevels();
+
+}

src/libutil/local.mk

@@ -7,3 +7,7 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+
+ifeq ($(HAVE_LIBCPUID), 1)
+	libutil_LDFLAGS += -lcpuid
+endif

src/libutil/util.cc

@@ -946,7 +946,7 @@ void killUser(uid_t uid)
 #else
             if (kill(-1, SIGKILL) == 0) break;
 #endif
-            if (errno == ESRCH) break; /* no more processes */
+            if (errno == ESRCH || errno == EPERM) break; /* no more processes */
             if (errno != EINTR)
                 throw SysError("cannot kill processes for uid '%1%'", uid);
         }

src/nix-build/nix-build.cc

@@ -240,8 +240,6 @@ static void main_nix_build(int argc, char * * argv)
 
     myArgs.parseCmdline(args);
 
-    initPlugins();
-
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
@@ -449,6 +447,7 @@ static void main_nix_build(int argc, char * * argv)
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "
                 "unset TZ; %6%"
+                "shopt -s execfail;"
                 "%7%",
                 shellEscape(tmpDir),
                 (pure ? "" : "p=$PATH; "),
@@ -518,9 +517,11 @@ static void main_nix_build(int argc, char * * argv)
             if (counter)
                 drvPrefix += fmt("-%d", counter + 1);
 
-            auto builtOutputs = store->queryDerivationOutputMap(drvPath);
+            auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath);
 
-            auto outputPath = builtOutputs.at(outputName);
+            auto maybeOutputPath = builtOutputs.at(outputName);
+            assert(maybeOutputPath);
+            auto outputPath = *maybeOutputPath;
 
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
                 std::string symlink = drvPrefix;

src/nix-channel/nix-channel.cc

@@ -196,8 +196,6 @@ static int main_nix_channel(int argc, char ** argv)
             return true;
         });
 
-        initPlugins();
-
         switch (cmd) {
             case cNone:
                 throw UsageError("no command specified");

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -74,8 +74,6 @@ static int main_nix_collect_garbage(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         auto profilesDir = settings.nixStateDir + "/profiles";
         if (removeOld) removeOldGenerations(profilesDir);
 

src/nix-copy-closure/nix-copy-closure.cc

@@ -43,8 +43,6 @@ static int main_nix_copy_closure(int argc, char ** argv)
             return true;
         });
 
-        initPlugins();
-
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
@@ -52,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv)
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 
-        StorePathSet storePaths2;
+        RealisedPath::Set storePaths2;
         for (auto & path : storePaths)
             storePaths2.insert(from->followLinksToStorePath(path));
 
-        StorePathSet closure;
+        RealisedPath::Set closure;
-        from->computeFSClosure(storePaths2, closure, false, includeOutputs);
+        RealisedPath::closure(*from, storePaths2, closure);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
 

src/nix-env/nix-env.cc

@@ -1420,8 +1420,6 @@ static int main_nix_env(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();

src/nix-instantiate/nix-instantiate.cc

@@ -149,8 +149,6 @@ static int main_nix_instantiate(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 

src/nix-store/nix-store.cc

@@ -905,6 +905,10 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
                     out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
+                if (GET_PROTOCOL_MINOR(clientVersion >= 5)) {
+                    worker_proto::write(*store, out, status.builtOutputs);
+                }
+
 
                 break;
             }
@@ -1067,8 +1071,6 @@ static int main_nix_store(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         if (!op) throw UsageError("no operation specified");
 
         if (op != opDump && op != opRestore) /* !!! hack */

src/nix/app.cc

@@ -12,11 +12,16 @@ App Installable::toApp(EvalState & state)
 
     auto type = cursor->getAttr("type")->getString();
 
+    auto checkProgram = [&](const Path & program)
+    {
+        if (!state.store->isInStore(program))
+            throw Error("app program '%s' is not in the Nix store", program);
+    };
+
     if (type == "app") {
         auto [program, context] = cursor->getAttr("program")->getStringWithContext();
 
-        if (!state.store->isInStore(program))
+        checkProgram(program);
-            throw Error("app program '%s' is not in the Nix store", program);
 
         std::vector<StorePathWithOutputs> context2;
         for (auto & [path, name] : context)
@@ -33,9 +38,17 @@ App Installable::toApp(EvalState & state)
         auto outPath = cursor->getAttr(state.sOutPath)->getString();
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
+        auto aMeta = cursor->maybeGetAttr("meta");
+        auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
+        auto mainProgram =
+            aMainProgram
+            ? aMainProgram->getString()
+            : DrvName(name).name;
+        auto program = outPath + "/bin/" + mainProgram;
+        checkProgram(program);
         return App {
             .context = { { drvPath, {outputName} } },
-            .program = outPath + "/bin/" + DrvName(name).name,
+            .program = program,
         };
     }
 

src/nix/bundle.cc

@@ -74,7 +74,7 @@ struct CmdBundle : InstallableCommand
 
         auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
-        auto bundler = InstallableFlake(
+        auto bundler = InstallableFlake(this,
             evalState, std::move(bundlerFlakeRef),
             Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
             Strings({"bundlers."}), lockFlags);

src/nix/copy.cc

@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdCopy : StorePathsCommand
+struct CmdCopy : RealisedPathsCommand
 {
     std::string srcUri, dstUri;
 
@@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand
 
     SubstituteFlag substitute = NoSubstitute;
 
-    using StorePathsCommand::run;
+    using RealisedPathsCommand::run;
 
     CmdCopy()
-        : StorePathsCommand(true)
+        : RealisedPathsCommand(true)
     {
         addFlag({
             .longName = "from",
@@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand
         if (srcUri.empty() && dstUri.empty())
             throw UsageError("you must pass '--from' and/or '--to'");
 
-        StorePathsCommand::run(store);
+        RealisedPathsCommand::run(store);
     }
 
-    void run(ref<Store> srcStore, StorePaths storePaths) override
+    void run(ref<Store> srcStore, std::vector<RealisedPath> paths) override
     {
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
+        copyPaths(
+            srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()),
             NoRepair, checkSigs, substitute);
     }
 };

src/nix/daemon.cc

@@ -326,8 +326,6 @@ static int main_nix_daemon(int argc, char * * argv)
             return true;
         });
 
-        initPlugins();
-
         runDaemon(stdio);
 
         return 0;

src/nix/develop.cc

@@ -59,7 +59,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
 
     static std::string squotedStringRegex =
-        R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
+        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
 
     static std::string indexedArrayRegex =
         R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
@@ -443,6 +443,7 @@ struct CmdDevelop : Common, MixEnvironment
             auto state = getEvalState();
 
             auto bashInstallable = std::make_shared<InstallableFlake>(
+                this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},

src/nix/flake-lock.md

@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Update the `nixpkgs` and `nix` inputs of the flake in the current
+  directory:
+
+  ```console
+  # nix flake lock --update-input nixpkgs --update-input nix
+  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+  ```
+
+# Description
+
+This command updates the lock file of a flake (`flake.lock`) so that
+it contains a lock for every flake input specified in
+`flake.nix`. Existing lock file entries are not updated unless
+required by a flag such as `--update-input`.
+
+Note that every command that operates on a flake will also update the
+lock file if needed, and supports the same flags. Therefore,
+
+```console
+# nix flake lock --update-input nixpkgs
+# nix build
+```
+
+is equivalent to:
+
+```console
+# nix build --update-input nixpkgs
+```
+
+Thus, this command is only useful if you want to update the lock file
+separately from any other action such as building.
+
+)""

src/nix/flake-update.md

@@ -2,52 +2,33 @@ R""(
 
 # Examples
 
-* Update the `nixpkgs` and `nix` inputs of the flake in the current
-  directory:
-
-  ```console
-  # nix flake update --update-input nixpkgs --update-input nix
-  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
-  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
-  ```
-
 * Recreate the lock file (i.e. update all inputs) and commit the new
   lock file:
 
   ```console
-  # nix flake update --recreate-lock-file --commit-lock-file
+  # nix flake update
+  * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+  * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
   …
   warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
   ```
 
 # Description
 
-This command updates the lock file of a flake (`flake.lock`) so that
+This command recreates the lock file of a flake (`flake.lock`), thus
-it contains a lock for every flake input specified in
+updating the lock for every mutable input (like `nixpkgs`) to its
-`flake.nix`. Note that every command that operates on a flake will
+current version. This is equivalent to passing `--recreate-lock-file`
-also update the lock file if needed, and supports the same
+to any command that operates on a flake. That is,
-flags. Therefore,
 
 ```console
-# nix flake update --update-input nixpkgs
+# nix flake update
 # nix build
 ```
 
 is equivalent to:
 
 ```console
-# nix build --update-input nixpkgs
+# nix build --recreate-lock-file
 ```
 
-Thus, this command is only useful if you want to update the lock file
-separately from any other action such as building.
-
-> **Note**
->
-> This command does *not* update locks that are already present unless
-> you explicitly ask for it using `--update-input` or
-> `--recreate-lock-file`. Thus, if the lock file already has locks for
-> every input, then `nix flake update` (without arguments) does
-> nothing.
-
 )""

src/nix/flake.cc

@@ -104,6 +104,14 @@ struct CmdFlakeUpdate : FlakeCommand
         return "update flake lock file";
     }
 
+    CmdFlakeUpdate()
+    {
+        /* Remove flags that don't make sense. */
+        removeFlag("recreate-lock-file");
+        removeFlag("update-input");
+        removeFlag("no-update-lock-file");
+    }
+
     std::string doc() override
     {
         return
@@ -113,7 +121,30 @@ struct CmdFlakeUpdate : FlakeCommand
 
     void run(nix::ref<nix::Store> store) override
     {
-        /* Use --refresh by default for 'nix flake update'. */
+        settings.tarballTtl = 0;
+
+        lockFlags.recreateLockFile = true;
+
+        lockFlake();
+    }
+};
+
+struct CmdFlakeLock : FlakeCommand
+{
+    std::string description() override
+    {
+        return "create missing lock file entries";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "flake-lock.md"
+          ;
+    }
+
+    void run(nix::ref<nix::Store> store) override
+    {
         settings.tarballTtl = 0;
 
         lockFlake();
@@ -595,7 +626,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
 
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
-        auto installable = InstallableFlake(
+        auto installable = InstallableFlake(nullptr,
             evalState, std::move(templateFlakeRef),
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
@@ -880,7 +911,8 @@ struct CmdFlakeShow : FlakeCommand
                             || attrPath[0] == "nixosConfigurations"
                             || attrPath[0] == "nixosModules"
                             || attrPath[0] == "defaultApp"
-                            || attrPath[0] == "templates"))
+                            || attrPath[0] == "templates"
+                            || attrPath[0] == "overlays"))
                     || ((attrPath.size() == 1 || attrPath.size() == 2)
                         && (attrPath[0] == "checks"
                             || attrPath[0] == "packages"
@@ -943,7 +975,8 @@ struct CmdFlakeShow : FlakeCommand
                 else {
                     logger->cout("%s: %s",
                         headerPrefix,
-                        attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" :
+                        (attrPath.size() == 1 && attrPath[0] == "overlay")
+                        || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" :
                         attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" :
                         attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" :
                         ANSI_YELLOW "unknown" ANSI_NORMAL);
@@ -1004,6 +1037,7 @@ struct CmdFlake : NixMultiCommand
     CmdFlake()
         : MultiCommand({
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
+                {"lock", []() { return make_ref<CmdFlakeLock>(); }},
                 {"info", []() { return make_ref<CmdFlakeInfo>(); }},
                 {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }},
                 {"check", []() { return make_ref<CmdFlakeCheck>(); }},

src/nix/main.cc

@@ -17,6 +17,10 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
+#if __linux__
+#include <sys/resource.h>
+#endif
+
 #include <nlohmann/json.hpp>
 
 extern std::string chrootHelperName;
@@ -61,6 +65,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     bool printBuildLogs = false;
     bool useNet = true;
     bool refresh = false;
+    bool showVersion = false;
 
     NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
@@ -87,11 +92,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "version",
             .description = "Show version information.",
-            .handler = {[&]() { if (!completions) printVersion(programName); }},
+            .handler = {[&]() { showVersion = true; }},
         });
 
         addFlag({
             .longName = "offline",
+            .aliases = {"no-net"}, // FIXME: remove
             .description = "Disable substituters and consider all previously downloaded files up-to-date.",
             .handler = {[&]() { useNet = false; }},
         });
@@ -153,6 +159,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
           #include "nix.md"
           ;
     }
+
+    // Plugins may add new subcommands.
+    void pluginsInited() override
+    {
+        commands = RegisterCommand::getCommandsFor({});
+    }
 };
 
 static void showHelp(std::vector<std::string> subcommand)
@@ -277,7 +289,10 @@ void mainWrapped(int argc, char * * argv)
 
     if (completions) return;
 
-    initPlugins();
+    if (args.showVersion) {
+        printVersion(programName);
+        return;
+    }
 
     if (!args.command)
         throw UsageError("no subcommand specified");
@@ -318,6 +333,17 @@ void mainWrapped(int argc, char * * argv)
 
 int main(int argc, char * * argv)
 {
+    // Increase the default stack size for the evaluator and for
+    // libstdc++'s std::regex.
+    #if __linux__
+    rlim_t stackSize = 64 * 1024 * 1024;
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+
     return nix::handleExceptions(argv[0], [&]() {
         nix::mainWrapped(argc, argv);
     });

src/nix/prefetch.cc

@@ -171,8 +171,6 @@ static int main_nix_prefetch_url(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
-        initPlugins();
-
         if (args.size() > 2)
             throw UsageError("too many arguments");
 

src/nix/profile.cc

@@ -399,7 +399,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 Activity act(*logger, lvlChatty, actUnknown,
                     fmt("checking '%s' for updates", element.source->attrPath));
 
-                InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags);
+                InstallableFlake installable(
+                    this,
+                    getEvalState(),
+                    FlakeRef(element.source->originalRef),
+                    {element.source->attrPath},
+                    {},
+                    lockFlags);
 
                 auto [attrPath, resolvedRef, drv] = installable.toDerivation();
 

src/nix/run.md

@@ -43,9 +43,10 @@ program specified by the app definition.
 
 If *installable* evaluates to a derivation, it will try to execute the
 program `<out>/bin/<name>`, where *out* is the primary output store
-path of the derivation and *name* is the name part of the value of the
+path of the derivation and *name* is the `meta.mainProgram` attribute
-`name` attribute of the derivation (e.g. if `name` is set to
+of the derivation if it exists, and otherwise the name part of the
-`hello-1.10`, it will run `$out/bin/hello`).
+value of the `name` attribute of the derivation (e.g. if `name` is set
+to `hello-1.10`, it will run `$out/bin/hello`).
 
 # Flake output attributes
 

tests/build-hook-ca.nix

@@ -11,6 +11,7 @@ let
       args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
+      __contentAddressed = true;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 
@@ -19,7 +20,6 @@ let
     name = "build-remote-input-1";
     buildCommand = "echo FOO > $out";
     requiredSystemFeatures = ["foo"];
-    outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=";
   };
 
   input2 = mkDerivation {
@@ -27,7 +27,16 @@ let
     name = "build-remote-input-2";
     buildCommand = "echo BAR > $out";
     requiredSystemFeatures = ["bar"];
-    outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q=";
+  };
+
+  input3 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-3";
+    buildCommand = ''
+      read x < ${input2}
+      echo $x BAZ > $out
+    '';
+    requiredSystemFeatures = ["baz"];
   };
 
 in
@@ -38,8 +47,7 @@ in
     buildCommand =
       ''
         read x < ${input1}
-        read y < ${input2}
+        read y < ${input3}
         echo "$x $y" > $out
       '';
-    outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0=";
   }

tests/build-remote-content-addressed-fixed.sh

@@ -1,5 +0,0 @@
-source common.sh
-
-file=build-hook-ca.nix
-
-source build-remote.sh

tests/build-remote-content-addressed-floating.sh

@@ -0,0 +1,7 @@
+source common.sh
+
+file=build-hook-ca.nix
+
+sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
+
+source build-remote.sh

tests/ca/build.sh

@@ -48,6 +48,10 @@ testCutoff () {
 testGC () {
     nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
     nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
+    clearStore
+    buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA
+    nix-collect-garbage --experimental-features ca-derivations
+    buildAttr rootCA 1 -j0
 }
 
 testNixCommand () {
@@ -57,7 +61,9 @@ testNixCommand () {
 
 # Disabled until we have it properly working
 # testRemoteCache
+clearStore
 testDeterministicCA
+clearStore
 testCutoff
 testGC
 testNixCommand

tests/ca/common.sh

@@ -0,0 +1 @@
+source ../common.sh

tests/ca/content-addressed.nix

@@ -1,4 +1,4 @@
-with import ./config.nix;
+with import ../config.nix;
 
 { seed ? 0 }:
 # A simple content-addressed derivation.

tests/ca/nix-copy.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# Globally enable the ca derivations experimental flag
+sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf"
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+ensureCorrectlyCopied () {
+    attrPath="$1"
+    nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath"
+}
+
+testOneCopy () {
+    clearStore
+    rm -rf "$REMOTE_STORE_DIR"
+
+    attrPath="$1"
+    nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix
+
+    ensureCorrectlyCopied "$attrPath"
+
+    # Ensure that we can copy back what we put in the store
+    clearStore
+    nix copy --from $REMOTE_STORE \
+        --file ./content-addressed.nix "$attrPath" \
+        --no-check-sigs
+}
+
+for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do
+    testOneCopy "$attrPath"
+done

tests/common.sh.in

@@ -11,7 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
-export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
+export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
 unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then

tests/compute-levels.sh

@@ -0,0 +1,7 @@
+source common.sh
+
+if [[ $(uname -ms) = "Linux x86_64" ]]; then
+    # x86_64 CPUs must always support the baseline
+    # microarchitecture level.
+    nix -vv --version | grep -q "x86_64-v1-linux"
+fi

tests/flakes.sh

@@ -232,7 +232,7 @@ nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2
 nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh
 
 # Updating the flake should not change the lockfile.
-nix flake update $flake2Dir
+nix flake lock $flake2Dir
 [[ -z $(git -C $flake2Dir diff master) ]]
 
 # Now we should be able to build the flake in pure mode.
@@ -354,10 +354,10 @@ nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord
 nix build -o $TEST_ROOT/result flake4#xyzzy
 
 # Test 'nix flake update' and --override-flake.
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ -z $(git -C $flake3Dir diff master) ]]
 
-nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs
+nix flake update $flake3Dir --override-flake flake2 nixpkgs
 [[ ! -z $(git -C $flake3Dir diff master) ]]
 
 # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
@@ -389,7 +389,7 @@ cat > $flake3Dir/flake.nix <<EOF
   };
 }
 EOF
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 git -C $flake3Dir add flake.nix flake.lock
 git -C $flake3Dir commit -m 'Remove packages.xyzzy'
 git -C $flake3Dir checkout master
@@ -547,7 +547,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["foo"]' ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -559,7 +559,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2","flake1"]' ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -571,7 +571,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2"]' ]]
 
 # Test overriding inputs of inputs.
@@ -587,7 +587,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq .nodes.flake1.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
 
 cat > $flake3Dir/flake.nix <<EOF
@@ -600,7 +600,7 @@ cat > $flake3Dir/flake.nix <<EOF
 }
 EOF
 
-nix flake update $flake3Dir --recreate-lock-file
+nix flake update $flake3Dir
 [[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]]
 [[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
 
@@ -658,20 +658,20 @@ nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ
 
 # Test --override-input.
 git -C $flake3Dir reset --hard
-nix flake update $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
+nix flake lock $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
 [[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]]
 
-nix flake update $flake3Dir --override-input flake2/flake1 flake1
+nix flake lock $flake3Dir --override-input flake2/flake1 flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
 
-nix flake update $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
+nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]]
 
 # Test --update-input.
-nix flake update $flake3Dir
+nix flake lock $flake3Dir
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]]
 
-nix flake update $flake3Dir --update-input flake2/flake1
+nix flake lock $flake3Dir --update-input flake2/flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
 
 # Test 'nix flake list-inputs'.

tests/local.mk

@@ -17,6 +17,7 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote-input-addressed.sh \
+  build-remote-content-addressed-floating.sh \
   ssh-relay.sh \
   nar-access.sh \
   structured-attrs.sh \
@@ -37,10 +38,11 @@ nix_tests = \
   recursive.sh \
   describe-stores.sh \
   flakes.sh \
-  content-addressed.sh \
+  build.sh \
-  build.sh
+  compute-levels.sh \
+  ca/build.sh \
+  ca/nix-copy.sh
   # parallel.sh
-  # build-remote-content-addressed-fixed.sh \
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 

tests/plugins.sh

@@ -2,6 +2,6 @@ source common.sh
 
 set -o pipefail
 
-res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*)
+res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull)
 
 [ "$res"x = "nullx" ]