From a7d2a3d087f21c004716808c94c63c387b2e689b Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Wed, 16 Nov 2022 15:23:59 +0100
Subject: [PATCH 001/183] Allow system certs access to fixed-output derivations

---
 src/libstore/build/local-derivation-goal.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 5cea3b590..a4ebd244f 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1715,6 +1715,8 @@ void LocalDerivationGoal::runChild()
                 for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
                     if (pathExists(path))
                         ss.push_back(path);
+
+                dirsInChroot.emplace(settings.caFile, "/etc/ssl/certs/ca-certificates.crt");
             }
 
             for (auto & i : ss) dirsInChroot.emplace(i, i);

From a6f85e052c916f1589414be1a8ba999aff993296 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 13 Apr 2023 10:38:35 -0400
Subject: [PATCH 002/183] Support `repairPath` on most stores.

More progress on issue #5729

The method trivially generalizes to be store-implementation-agnostic, in
fact.

However, we force it to continue to be unimplemented with `RemoteStore`
and `LegacySSHStore` because the implementation we'd get via the
generalization is probably not the one users expect. This keeps our
hands untied to do it right going forward.

For more about the tension between the scheduler logic being
store-type-agnostic and remote stores doing their own scheduling, see
issues #5025 and #5056.
---
 src/libstore/build/entry-points.cc |  2 +-
 src/libstore/legacy-ssh-store.cc   | 11 +++++++++++
 src/libstore/local-store.hh        |  2 --
 src/libstore/remote-store.hh       | 11 +++++++++++
 src/libstore/store-api.hh          |  3 +--
 5 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 2925fe3ca..efa1445fa 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -119,7 +119,7 @@ void Store::ensurePath(const StorePath & path)
 }
 
 
-void LocalStore::repairPath(const StorePath & path)
+void Store::repairPath(const StorePath & path)
 {
     Worker worker(*this, *this);
     GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index d2ddbbe5f..2341d4cb2 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -345,6 +345,17 @@ public:
     virtual ref<FSAccessor> getFSAccessor() override
     { unsupported("getFSAccessor"); }
 
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
     void computeFSClosure(const StorePathSet & paths,
         StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false) override
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 55add18dd..70debad38 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -240,8 +240,6 @@ public:
 
     void vacuumDB();
 
-    void repairPath(const StorePath & path) override;
-
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
     /**
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 1c45f543e..0da039837 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -136,6 +136,17 @@ public:
 
     bool verifyStore(bool checkContents, RepairFlag repair) override;
 
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
     void queryMissing(const std::vector<DerivedPath> & targets,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5bee272bf..30c821c58 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -684,8 +684,7 @@ public:
      * Repair the contents of the given path by redownloading it using
      * a substituter (if available).
      */
-    virtual void repairPath(const StorePath & path)
-    { unsupported("repairPath"); }
+    virtual void repairPath(const StorePath & path);
 
     /**
      * Add signatures to the specified store path. The signatures are

From 0e18254aa81b9315c13d6ae736cb38666d19f122 Mon Sep 17 00:00:00 2001
From: Moritz Angermann <moritz.angermann@gmail.com>
Date: Wed, 5 Apr 2023 17:05:17 +0800
Subject: [PATCH 003/183] Fix shutdown behavior and resource management for
 recursive-nix on macOS

Previously, we relied on the `shutdown()` function to terminate `accept()`
calls on a listening socket. However, this approach did not work on macOS as
the waiting `accept()` call is not considered a connected socket, resulting in
an `ENOTCONN` error. Instead, we now close the listening socket to terminate
the `accept()` call.

Additionally, we fixed a resource management issue where we set the
`daemonSocket` variable to -1, triggering resource cleanup and causing the
`stopDaemon` function to be called twice. This resulted in errors as the socket
was already closed by the time the second `stopDaemon` call was made. Instead of
setting `daemonSocket` to -1, we now release the socket using the `release()`
method on a unique pointer. This properly transfers ownership and allows for
correct resource cleanup.

These changes ensure proper behavior and resource management for the
recursive-nix feature on macOS.
---
 src/libstore/build/local-derivation-goal.cc | 23 +++++++++++++++++----
 tests/recursive.sh                          |  3 ---
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 21cd6e7ee..4b978f2a4 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1457,7 +1457,7 @@ void LocalDerivationGoal::startDaemon()
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             if (!remote) {
                 if (errno == EINTR || errno == EAGAIN) continue;
-                if (errno == EINVAL) break;
+                if (errno == EINVAL || errno == ECONNABORTED) break;
                 throw SysError("accepting connection");
             }
 
@@ -1487,8 +1487,22 @@ void LocalDerivationGoal::startDaemon()
 
 void LocalDerivationGoal::stopDaemon()
 {
-    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
-        throw SysError("shutting down daemon socket");
+    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) {
+        // According to the POSIX standard, the 'shutdown' function should
+        // return an ENOTCONN error when attempting to shut down a socket that
+        // hasn't been connected yet. This situation occurs when the 'accept'
+        // function is called on a socket without any accepted connections,
+        // leaving the socket unconnected. While Linux doesn't seem to produce
+        // an error for sockets that have only been accepted, more
+        // POSIX-compliant operating systems like OpenBSD, macOS, and others do
+        // return the ENOTCONN error. Therefore, we handle this error here to
+        // avoid raising an exception for compliant behaviour.
+        if (errno == ENOTCONN) {
+            daemonSocket.close();
+        } else {
+            throw SysError("shutting down daemon socket");
+        }
+    }
 
     if (daemonThread.joinable())
         daemonThread.join();
@@ -1499,7 +1513,8 @@ void LocalDerivationGoal::stopDaemon()
         thread.join();
     daemonWorkerThreads.clear();
 
-    daemonSocket = -1;
+    // release the socket.
+    daemonSocket.close();
 }
 
 
diff --git a/tests/recursive.sh b/tests/recursive.sh
index 638f06f85..b661422ed 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -3,9 +3,6 @@ source common.sh
 sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf
 restartDaemon
 
-# FIXME
-if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
-
 clearStore
 
 rm -f $TEST_ROOT/result

From 64b0cc5bc40db052a4eff15c6d1ed8e1efafcd0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 3 Apr 2023 15:13:11 +0200
Subject: [PATCH 004/183] Make `nix-collect-garbage -d` look into more places

Make it look into the new-style profiles dir, the old-style one, and the
target of `~/.nix-profile` to be sure that we don't miss anything
---
 src/nix-collect-garbage/nix-collect-garbage.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 3cc57af4e..cb1f42e35 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -77,7 +77,12 @@ static int main_nix_collect_garbage(int argc, char * * argv)
             return true;
         });
 
-        if (removeOld) removeOldGenerations(profilesDir());
+        if (removeOld) {
+            std::set<Path> dirsToClean = {
+                profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())};
+            for (auto & dir : dirsToClean)
+                removeOldGenerations(dir);
+        }
 
         // Run the actual garbage collector.
         if (!dryRun) {

From e97e9e9f0055d02c16a3dc8e7891c06d643aaf82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 15 May 2023 11:58:49 +0200
Subject: [PATCH 005/183] test the garbage collection with the old profile dir

Regression test for https://github.com/NixOS/nix/issues/8294
---
 tests/gc.sh | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/tests/gc.sh b/tests/gc.sh
index 98d6cb032..95669e25c 100644
--- a/tests/gc.sh
+++ b/tests/gc.sh
@@ -52,9 +52,7 @@ rmdir $NIX_STORE_DIR/.links
 rmdir $NIX_STORE_DIR
 
 ## Test `nix-collect-garbage -d`
-# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
-# using them
-if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+testCollectGarbageD () {
     clearProfiles
     # Run two `nix-env` commands, should create two generations of
     # the profile
@@ -66,4 +64,17 @@ if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
     # left
     nix-collect-garbage -d
     [[ $(nix-env --list-generations | wc -l) -eq 1 ]]
+}
+# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
+# using them
+if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+    testCollectGarbageD
+
+    # Run the same test, but forcing the profiles at their legacy location under
+    # /nix/var/nix.
+    #
+    # Regression test for #8294
+    rm ~/.nix-profile
+    ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile
+    testCollectGarbageD
 fi

From 275468a1f0f9c0037d79b7adee68a2ae16c0af4a Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 15 May 2023 15:26:14 +0200
Subject: [PATCH 006/183] list files used by `nix-channel`

---
 doc/manual/src/command-ref/nix-channel.md | 6 ++++++
 doc/manual/src/command-ref/nix-env.md     | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index 72d3e422b..d2144b917 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -52,6 +52,12 @@ The list of subscribed channels is stored in `~/.nix-channels`.
 
 {{#include ./env-common.md}}
 
+# Files
+
+`nix-channel` operates on the following files.
+
+{{#include ./files/channels.md}}
+
 # Examples
 
 To subscribe to the Nixpkgs channel and install the GNU Hello package:
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index b4a3dce49..650fea55c 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -83,6 +83,8 @@ match. Here are some examples:
 
 # Files
 
+`nix-env` operates on the following files.
+
 {{#include ./files/default-nix-expression.md}}
 
 {{#include ./files/profiles.md}}

From 8976769a1c4a7fa6276d38402d97cec89b27d98f Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 16 May 2023 05:30:20 +0200
Subject: [PATCH 007/183] Reword info on XDG base dirs (#8338)

This gives some more context and should clarify why it works that way.
Also link it from the section on `NIX_USER_CONF_FILES`.

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 doc/manual/src/command-ref/env-common.md | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index bf00be84f..b4a9bb2a9 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -71,9 +71,12 @@ Most Nix commands interpret the following environment variables:
     Settings are separated by the newline character.
 
   - <span id="env-NIX_USER_CONF_FILES">[`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)</span>\
-    Overrides the location of the user Nix configuration files to load
-    from (defaults to the XDG spec locations). The variable is treated
-    as a list separated by the `:` token.
+    Overrides the location of the Nix user configuration files to load from.
+
+    The default are the locations according to the [XDG Base Directory Specification].
+    See the [XDG Base Directories](#xdg-base-directories) sub-section for details.
+
+    The variable is treated as a list separated by the `:` token.
 
   - <span id="env-TMPDIR">[`TMPDIR`](#env-TMPDIR)</span>\
     Use the specified directory to store temporary files. In particular,
@@ -103,15 +106,19 @@ Most Nix commands interpret the following environment variables:
     384 MiB. Setting it to a low value reduces memory consumption, but
     will increase runtime due to the overhead of garbage collection.
 
-## XDG Base Directory
+## XDG Base Directories
 
-New Nix commands conform to the [XDG Base Directory Specification], and use the following environment variables to determine locations of various state and configuration files:
+Nix follows the [XDG Base Directory Specification].
+
+For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled.
+[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default.
+
+The following environment variables are used to determine locations of various state and configuration files:
 
 - [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`)
 - [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`)
 - [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`)
 
-Classic Nix commands can also be made to follow this standard using the [`use-xdg-base-directories`] configuration option.
 
 [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
-[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories
\ No newline at end of file
+[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories

From 3d144e778e4193c948fa6e0a23f9f3de194bc62f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Tue, 16 May 2023 11:31:28 +0200
Subject: [PATCH 008/183] Rename `ProfileElement.describe()` to `.identifier`

This method isn't used to describe what the element is, but to return a
unique identifier for it whithin the current profile
---
 src/nix/profile.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index fd63b3519..ce3b6fbbc 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -42,7 +42,7 @@ struct ProfileElement
     bool active = true;
     int priority = defaultPriority;
 
-    std::string describe() const
+    std::string identifier() const
     {
         if (source)
             return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string());
@@ -62,7 +62,7 @@ struct ProfileElement
 
     bool operator < (const ProfileElement & other) const
     {
-        return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
+        return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths);
     }
 
     void updateStorePaths(
@@ -237,13 +237,13 @@ struct ProfileManifest
         bool changes = false;
 
         while (i != prevElems.end() || j != curElems.end()) {
-            if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) {
-                logger->cout("%s%s: ∅ -> %s", indent, j->describe(), j->versions());
+            if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) {
+                logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions());
                 changes = true;
                 ++j;
             }
-            else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) {
-                logger->cout("%s%s: %s -> ∅", indent, i->describe(), i->versions());
+            else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) {
+                logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions());
                 changes = true;
                 ++i;
             }
@@ -251,7 +251,7 @@ struct ProfileManifest
                 auto v1 = i->versions();
                 auto v2 = j->versions();
                 if (v1 != v2) {
-                    logger->cout("%s%s: %s -> %s", indent, i->describe(), v1, v2);
+                    logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2);
                     changes = true;
                 }
                 ++i;
@@ -491,7 +491,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
             if (!matches(*store, element, i, matchers)) {
                 newManifest.elements.push_back(std::move(element));
             } else {
-                notice("removing '%s'", element.describe());
+                notice("removing '%s'", element.identifier());
             }
         }
 

From e997512523d8c5d22005b6cd5e22271284273ee6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Tue, 16 May 2023 11:37:45 +0200
Subject: [PATCH 009/183] Fix the printing of the installables on `nix profile
 install` conflict

- If the element comes from a flake, print the full flakeref (with the
  fragment part) and not just the reference to the flake itself
- If the element doesn't come from a flake, print its store path(s)

This is a bit too verbose, but has the advantages of being correct (and
not crashing), so it's strictly better than the previous situation

Fix https://github.com/NixOS/nix/issues/8284
---
 src/nix/profile.cc   | 33 ++++++++++++++++++++++++++-------
 tests/nix-profile.sh | 13 ++++++++++---
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ce3b6fbbc..7cea616d2 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -31,6 +31,11 @@ struct ProfileElementSource
             std::tuple(originalRef.to_string(), attrPath, outputs) <
             std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs);
     }
+
+    std::string to_string() const
+    {
+        return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string());
+    }
 };
 
 const int defaultPriority = 5;
@@ -45,13 +50,27 @@ struct ProfileElement
     std::string identifier() const
     {
         if (source)
-            return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string());
+            return source->to_string();
         StringSet names;
         for (auto & path : storePaths)
             names.insert(DrvName(path.name()).name);
         return concatStringsSep(", ", names);
     }
 
+    /**
+     * Return a string representing an installable corresponding to the current
+     * element, either a flakeref or a plain store path
+     */
+    std::set<std::string> toInstallables(Store & store)
+    {
+        if (source)
+            return {source->to_string()};
+        StringSet rawPaths;
+        for (auto & path : storePaths)
+            rawPaths.insert(store.printStorePath(path));
+        return rawPaths;
+    }
+
     std::string versions() const
     {
         StringSet versions;
@@ -363,10 +382,10 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     auto profileElement = *it;
                     for (auto & storePath : profileElement.storePaths) {
                         if (conflictError.fileA.starts_with(store->printStorePath(storePath))) {
-                            return std::pair(conflictError.fileA, profileElement.source->originalRef);
+                            return std::pair(conflictError.fileA, profileElement.toInstallables(*store));
                         }
                         if (conflictError.fileB.starts_with(store->printStorePath(storePath))) {
-                            return std::pair(conflictError.fileB, profileElement.source->originalRef);
+                            return std::pair(conflictError.fileB, profileElement.toInstallables(*store));
                         }
                     }
                 }
@@ -375,9 +394,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             // There are 2 conflicting files. We need to find out which one is from the already installed package and
             // which one is the package that is the new package that is being installed.
             // The first matching package is the one that was already installed (original).
-            auto [originalConflictingFilePath, originalConflictingRef] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
+            auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
             // The last matching package is the one that was going to be installed (new).
-            auto [newConflictingFilePath, newConflictingRef] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
+            auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
 
             throw Error(
                 "An existing package already provides the following file:\n"
@@ -403,8 +422,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 "  nix profile install %4% --priority %7%\n",
                 originalConflictingFilePath,
                 newConflictingFilePath,
-                originalConflictingRef.to_string(),
-                newConflictingRef.to_string(),
+                concatStringsSep(" ", originalConflictingRefs),
+                concatStringsSep(" ", newConflictingRefs),
                 conflictError.priority,
                 conflictError.priority - 1,
                 conflictError.priority + 1
diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh
index 4ef5b484a..9da3f802b 100644
--- a/tests/nix-profile.sh
+++ b/tests/nix-profile.sh
@@ -157,17 +157,17 @@ error: An existing package already provides the following file:
 
        To remove the existing package:
 
-         nix profile remove path:${flake1Dir}
+         nix profile remove path:${flake1Dir}#packages.${system}.default
 
        The new package can also be installed next to the existing one by assigning a different priority.
        The conflicting packages have a priority of 5.
        To prioritise the new package:
 
-         nix profile install path:${flake2Dir} --priority 4
+         nix profile install path:${flake2Dir}#packages.${system}.default --priority 4
 
        To prioritise the existing package:
 
-         nix profile install path:${flake2Dir} --priority 6
+         nix profile install path:${flake2Dir}#packages.${system}.default --priority 6
 EOF
 )
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
@@ -177,3 +177,10 @@ nix profile install $flake2Dir --priority 0
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]]
 # nix profile install $flake1Dir --priority 100
 # [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+
+# Ensure that conflicts are handled properly even when the installables aren't
+# flake references.
+# Regression test for https://github.com/NixOS/nix/issues/8284
+clearProfiles
+nix profile install $(nix build $flake1Dir --no-link --print-out-paths)
+expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default"

From 908bc9a9574d31193b90e6c9ceb65d9622d4ed78 Mon Sep 17 00:00:00 2001
From: "Markus S. Wamser" <github-dev@mail2013.wamser.eu>
Date: Fri, 28 Apr 2023 22:24:47 +0200
Subject: [PATCH 010/183] fix "add an option to include flake-registry..."

commit 6dbce3215fa2e30e1daafcc70d6926cd97987612 was missing a pair of
parentheses
---
 docker.nix | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/docker.nix b/docker.nix
index 52199af66..bd16b71cd 100644
--- a/docker.nix
+++ b/docker.nix
@@ -190,6 +190,12 @@ let
         cp -a ${rootEnv}/* $out/
         ln -s ${manifest} $out/manifest.nix
       '';
+      flake-registry-path = if (flake-registry == null) then
+        null
+      else if (builtins.readFileType (toString flake-registry)) == "directory" then
+        "${flake-registry}/flake-registry.json"
+      else
+        flake-registry;
     in
     pkgs.runCommand "base-system"
       {
@@ -202,7 +208,7 @@ let
         ];
         allowSubstitutes = false;
         preferLocalBuild = true;
-      } ''
+      } (''
       env
       set -x
       mkdir -p $out/etc
@@ -249,15 +255,15 @@ let
       ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
       ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
 
-    '' + (lib.optionalString (flake-registry != null) ''
+    '' + (lib.optionalString (flake-registry-path != null) ''
       nixCacheDir="/root/.cache/nix"
       mkdir -p $out$nixCacheDir
       globalFlakeRegistryPath="$nixCacheDir/flake-registry.json"
-      ln -s ${flake-registry}/flake-registry.json $out$globalFlakeRegistryPath
+      ln -s ${flake-registry-path} $out$globalFlakeRegistryPath
       mkdir -p $out/nix/var/nix/gcroots/auto
       rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath))
       ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
-    '');
+    ''));
 
 in
 pkgs.dockerTools.buildLayeredImageWithNixDb {

From 61cdb0b0576017a636eabb6899aac097bb7718c0 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Tue, 16 May 2023 16:48:04 +0400
Subject: [PATCH 011/183] Fix ControlMaster behaviour

---
 src/libstore/ssh.cc | 23 ++++++++++++++++-------
 src/libstore/ssh.hh |  1 +
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 6f6deda51..fae99d75b 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
     args.push_back("-oLocalCommand=echo started");
 }
 
+bool SSHMaster::isMasterRunning() {
+    auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
+    return res.first == 0;
+}
+
 std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
 {
     Path socketPath = startMaster();
@@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
     // Wait for the SSH connection to be established,
     // So that we don't overwrite the password prompt with our progress bar.
-    if (!fakeSSH && !useMaster) {
+    if (!fakeSSH && !useMaster && !isMasterRunning()) {
         std::string reply;
         try {
             reply = readLine(out.readSide.get());
@@ -133,6 +138,8 @@ Path SSHMaster::startMaster()
     logger->pause();
     Finally cleanup = [&]() { logger->resume(); };
 
+    bool wasMasterRunning = isMasterRunning();
+
     state->sshMaster = startProcess([&]() {
         restoreProcessContext();
 
@@ -152,13 +159,15 @@ Path SSHMaster::startMaster()
 
     out.writeSide = -1;
 
-    std::string reply;
-    try {
-        reply = readLine(out.readSide.get());
-    } catch (EndOfFile & e) { }
+    if (!wasMasterRunning) {
+        std::string reply;
+        try {
+            reply = readLine(out.readSide.get());
+        } catch (EndOfFile & e) { }
 
-    if (reply != "started")
-        throw Error("failed to start SSH master connection to '%s'", host);
+        if (reply != "started")
+            throw Error("failed to start SSH master connection to '%s'", host);
+    }
 
     return state->socketPath;
 }
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index c86a8a986..94b952af9 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -28,6 +28,7 @@ private:
     Sync<State> state_;
 
     void addCommonSSHOpts(Strings & args);
+    bool isMasterRunning();
 
 public:
 

From 25434df0d9e05bbaf7f7f881f2b53134c95c4665 Mon Sep 17 00:00:00 2001
From: Konstantin Vukolov <kasyanvukolov@yandex.ru>
Date: Wed, 17 May 2023 02:00:32 +0300
Subject: [PATCH 012/183] Ask for git credentials in fetcher

---
 src/libfetchers/git.cc | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 1da8c9609..7ec6efa4d 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -1,4 +1,5 @@
 #include "fetchers.hh"
+#include "finally.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
@@ -21,6 +22,14 @@ namespace nix::fetchers {
 
 namespace {
 
+template<typename... Args>
+auto runProgramWithCredentialsInput(Args... args)
+{
+    logger->pause();
+    Finally defer([]{ logger->resume(); });
+    return runProgram(std::forward<Args>(args)...);
+}
+
 // Explicit initial branch of our bare repo to suppress warnings from new version of git.
 // The value itself does not matter, since we always fetch a specific revision or branch.
 // It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
@@ -58,7 +67,7 @@ Path getCachePath(std::string_view key)
 //   ...
 std::optional<std::string> readHead(const Path & path)
 {
-    auto [status, output] = runProgram(RunOptions {
+    auto [status, output] = runProgramWithCredentialsInput(RunOptions {
         .program = "git",
         // FIXME: use 'HEAD' to avoid returning all refs
         .args = {"ls-remote", "--symref", path},
@@ -350,7 +359,7 @@ struct GitInputScheme : InputScheme
 
         args.push_back(destDir);
 
-        runProgram("git", true, args);
+        runProgramWithCredentialsInput("git", true, args);
     }
 
     std::optional<Path> getSourcePath(const Input & input) override
@@ -555,7 +564,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                    runProgramWithCredentialsInput("git", true, Strings { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -621,7 +630,7 @@ struct GitInputScheme : InputScheme
                 // exists, see FIXME above) so use a big hammer and fetch
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
+                runProgramWithCredentialsInput("git", true, Strings { "-C", tmpDir, "fetch", "--quiet", "--force",
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
             }
 
@@ -649,7 +658,7 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+                runProgramWithCredentialsInput("git", true, Strings{ "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
             }
 
             filter = isNotDotGitDirectory;

From 8d4b6766e297314f69ee96b6c9d74c6c2637f84b Mon Sep 17 00:00:00 2001
From: Alexander Schmolck <a.schmolck+git@gmail.com>
Date: Sun, 30 Apr 2023 14:52:38 +0100
Subject: [PATCH 013/183] Convert short nix options to long ones

e.g. nix-env -e subversion => nix-env --uninstall subversion

The aim is to make the documentation less cryptic for newcomers and the
long options are more self-documenting.

The change was made with the following script:

<https://github.com/aschmolck/convert-short-nix-opts-to-long-ones>

and sanity checked visually.
---
 doc/manual/src/advanced-topics/diff-hook.md   | 10 ++++----
 .../src/advanced-topics/post-build-hook.md    |  2 +-
 doc/manual/src/command-ref/nix-build.md       | 10 ++++----
 doc/manual/src/command-ref/nix-channel.md     |  6 ++---
 .../src/command-ref/nix-copy-closure.md       |  2 +-
 doc/manual/src/command-ref/nix-env.md         |  2 +-
 .../command-ref/nix-env/delete-generations.md |  2 +-
 doc/manual/src/command-ref/nix-env/install.md | 24 +++++++++----------
 doc/manual/src/command-ref/nix-env/query.md   | 16 ++++++-------
 .../src/command-ref/nix-env/set-flag.md       | 10 ++++----
 doc/manual/src/command-ref/nix-env/set.md     |  2 +-
 .../command-ref/nix-env/switch-generation.md  |  2 +-
 .../src/command-ref/nix-env/switch-profile.md |  2 +-
 .../src/command-ref/nix-env/uninstall.md      |  2 +-
 doc/manual/src/command-ref/nix-env/upgrade.md |  8 +++----
 doc/manual/src/command-ref/nix-instantiate.md | 16 ++++++-------
 doc/manual/src/command-ref/nix-shell.md       | 24 +++++++++----------
 doc/manual/src/command-ref/nix-store/dump.md  |  2 +-
 .../src/command-ref/nix-store/export.md       |  2 +-
 .../src/command-ref/nix-store/opt-common.md   |  2 +-
 doc/manual/src/command-ref/nix-store/query.md | 14 +++++------
 .../src/command-ref/nix-store/read-log.md     |  2 +-
 .../src/command-ref/nix-store/realise.md      |  4 ++--
 .../src/command-ref/nix-store/verify-path.md  |  2 +-
 doc/manual/src/command-ref/opt-common.md      |  6 ++---
 doc/manual/src/contributing/hacking.md        |  6 ++---
 doc/manual/src/glossary.md                    |  2 +-
 doc/manual/src/installation/upgrading.md      |  6 ++---
 doc/manual/src/introduction.md                |  6 ++---
 .../package-management/basic-package-mgmt.md  | 24 +++++++++----------
 .../binary-cache-substituter.md               |  4 ++--
 doc/manual/src/package-management/channels.md |  2 +-
 .../src/package-management/copy-closure.md    |  4 ++--
 doc/manual/src/package-management/profiles.md |  6 ++---
 .../src/package-management/ssh-substituter.md |  4 ++--
 35 files changed, 119 insertions(+), 119 deletions(-)

diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md
index 4a742c160..207aad3b8 100644
--- a/doc/manual/src/advanced-topics/diff-hook.md
+++ b/doc/manual/src/advanced-topics/diff-hook.md
@@ -48,13 +48,13 @@ If the build passes and is deterministic, Nix will exit with a status
 code of 0:
 
 ```console
-$ nix-build ./deterministic.nix -A stable
+$ nix-build ./deterministic.nix --attr stable
 this derivation will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
 
-$ nix-build ./deterministic.nix -A stable --check
+$ nix-build ./deterministic.nix --attr stable --check
 checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
 ```
@@ -63,13 +63,13 @@ If the build is not deterministic, Nix will exit with a status code of
 1:
 
 ```console
-$ nix-build ./deterministic.nix -A unstable
+$ nix-build ./deterministic.nix --attr unstable
 this derivation will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
 
-$ nix-build ./deterministic.nix -A unstable --check
+$ nix-build ./deterministic.nix --attr unstable --check
 checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may
 not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
@@ -89,7 +89,7 @@ Using `--check` with `--keep-failed` will cause Nix to keep the second
 build's output in a special, `.check` path:
 
 ```console
-$ nix-build ./deterministic.nix -A unstable --check --keep-failed
+$ nix-build ./deterministic.nix --attr unstable --check --keep-failed
 checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 note: keeping build directory '/tmp/nix-build-unstable.drv-0'
 error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may
diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md
index 1479cc3a4..a251dec48 100644
--- a/doc/manual/src/advanced-topics/post-build-hook.md
+++ b/doc/manual/src/advanced-topics/post-build-hook.md
@@ -90,7 +90,7 @@ Then, restart the `nix-daemon`.
 Build any derivation, for example:
 
 ```console
-$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
+$ nix-build --expr '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
 this derivation will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 44de4cf53..f70bbd7f2 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -76,7 +76,7 @@ except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`.
 # Examples
 
 ```console
-$ nix-build '<nixpkgs>' -A firefox
+$ nix-build '<nixpkgs>' --attr firefox
 store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv
 /nix/store/d18hyl92g30l...-firefox-1.5.0.7
 
@@ -91,7 +91,7 @@ If a derivation has multiple outputs, `nix-build` will build the default
 (first) output. You can also build all outputs:
 
 ```console
-$ nix-build '<nixpkgs>' -A openssl.all
+$ nix-build '<nixpkgs>' --attr openssl.all
 ```
 
 This will create a symlink for each output named `result-outputname`.
@@ -101,7 +101,7 @@ outputs `out`, `bin` and `man`, `nix-build` will create symlinks
 specific output:
 
 ```console
-$ nix-build '<nixpkgs>' -A openssl.man
+$ nix-build '<nixpkgs>' --attr openssl.man
 ```
 
 This will create a symlink `result-man`.
@@ -109,7 +109,7 @@ This will create a symlink `result-man`.
 Build a Nix expression given on the command line:
 
 ```console
-$ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
+$ nix-build --expr 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"'
 $ cat ./result
 bar
 ```
@@ -118,5 +118,5 @@ Build the GNU Hello package from the latest revision of the master
 branch of Nixpkgs:
 
 ```console
-$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz --attr hello
 ```
diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index 72d3e422b..cd86b364d 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -59,18 +59,18 @@ To subscribe to the Nixpkgs channel and install the GNU Hello package:
 ```console
 $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
 $ nix-channel --update
-$ nix-env -iA nixpkgs.hello
+$ nix-env --install --attr nixpkgs.hello
 ```
 
 You can revert channel updates using `--rollback`:
 
 ```console
-$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
 "14.04.527.0e935f1"
 
 $ nix-channel --rollback
 switching from generation 483 to 482
 
-$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
 "14.04.526.dbadfad"
 ```
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 0801e8246..fbf6828da 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -87,5 +87,5 @@ environment:
 ```console
 $ nix-copy-closure --from alice@itchy.labs \
     /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
-$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+$ nix-env --install /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
 ```
diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md
index b4a3dce49..486238ac8 100644
--- a/doc/manual/src/command-ref/nix-env.md
+++ b/doc/manual/src/command-ref/nix-env.md
@@ -49,7 +49,7 @@ These pages can be viewed offline:
 
 # Selectors
 
-Several commands, such as `nix-env -q` and `nix-env -i`, take a list of
+Several commands, such as `nix-env --query ` and `nix-env --install `, take a list of
 arguments that specify the packages on which to operate. These are
 extended regular expressions that must match the entire name of the
 package. (For details on regular expressions, see **regex**(7).) The match is
diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md
index 6f0af5384..92cb7f0d9 100644
--- a/doc/manual/src/command-ref/nix-env/delete-generations.md
+++ b/doc/manual/src/command-ref/nix-env/delete-generations.md
@@ -41,6 +41,6 @@ $ nix-env --delete-generations 30d
 ```
 
 ```console
-$ nix-env -p other_profile --delete-generations old
+$ nix-env --profile other_profile --delete-generations old
 ```
 
diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md
index d754accfe..ad179cbc7 100644
--- a/doc/manual/src/command-ref/nix-env/install.md
+++ b/doc/manual/src/command-ref/nix-env/install.md
@@ -36,7 +36,7 @@ a number of possible ways:
     then the derivation with the highest version will be installed.
 
     You can force the installation of multiple derivations with the same
-    name by being specific about the versions. For instance, `nix-env -i
+    name by being specific about the versions. For instance, `nix-env --install
     gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will
     probably cause a user environment conflict\!).
 
@@ -44,7 +44,7 @@ a number of possible ways:
     paths* that select attributes from the top-level Nix
     expression. This is faster than using derivation names and
     unambiguous. To find out the attribute paths of available
-    packages, use `nix-env -qaP`.
+    packages, use `nix-env --query --available --attr-path `.
 
   - If `--from-profile` *path* is given, *args* is a set of names
     denoting installed store paths in the profile *path*. This is an
@@ -87,7 +87,7 @@ a number of possible ways:
 
   - `--remove-all` / `-r`\
     Remove all previously installed packages first. This is equivalent
-    to running `nix-env -e '.*'` first, except that everything happens
+    to running `nix-env --uninstall '.*'` first, except that everything happens
     in a single transaction.
 
 {{#include ./opt-common.md}}
@@ -103,9 +103,9 @@ a number of possible ways:
 To install a package using a specific attribute path from the active Nix expression:
 
 ```console
-$ nix-env -iA gcc40mips
+$ nix-env --install --attr gcc40mips
 installing `gcc-4.0.2'
-$ nix-env -iA xorg.xorgserver
+$ nix-env --install --attr xorg.xorgserver
 installing `xorg-server-1.2.0'
 ```
 
@@ -133,32 +133,32 @@ installing `gcc-3.3.2'
 To install all derivations in the Nix expression `foo.nix`:
 
 ```console
-$ nix-env -f ~/foo.nix -i '.*'
+$ nix-env --file ~/foo.nix --install '.*'
 ```
 
 To copy the store path with symbolic name `gcc` from another profile:
 
 ```console
-$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc
+$ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc
 ```
 
 To install a specific [store derivation] (typically created by
 `nix-instantiate`):
 
 ```console
-$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
+$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
 ```
 
 To install a specific output path:
 
 ```console
-$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3
+$ nix-env --install /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3
 ```
 
 To install from a Nix expression specified on the command-line:
 
 ```console
-$ nix-env -f ./foo.nix -i -E \
+$ nix-env --file ./foo.nix --install --expr \
     'f: (f {system = "i686-linux";}).subversionWithJava'
 ```
 
@@ -170,7 +170,7 @@ function defined in `./foo.nix`.
 A dry-run tells you which paths will be downloaded or built from source:
 
 ```console
-$ nix-env -f '<nixpkgs>' -iA hello --dry-run
+$ nix-env --file '<nixpkgs>' --install --attr hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
 this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
@@ -182,6 +182,6 @@ To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12
 channel:
 
 ```console
-$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox
 ```
 
diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md
index 18f0ee210..c9b4d8513 100644
--- a/doc/manual/src/command-ref/nix-env/query.md
+++ b/doc/manual/src/command-ref/nix-env/query.md
@@ -137,7 +137,7 @@ derivation is shown unless `--no-name` is specified.
 To show installed packages:
 
 ```console
-$ nix-env -q
+$ nix-env --query
 bison-1.875c
 docbook-xml-4.2
 firefox-1.0.4
@@ -149,7 +149,7 @@ ORBit2-2.8.3
 To show available packages:
 
 ```console
-$ nix-env -qa
+$ nix-env --query --available
 firefox-1.0.7
 GConf-2.4.0.1
 MPlayer-1.0pre7
@@ -160,7 +160,7 @@ ORBit2-2.8.3
 To show the status of available packages:
 
 ```console
-$ nix-env -qas
+$ nix-env --query --available --status
 -P- firefox-1.0.7   (not installed but present)
 --S GConf-2.4.0.1   (not present, but there is a substitute for fast installation)
 --S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!)
@@ -171,14 +171,14 @@ IP- ORBit2-2.8.3    (installed and by definition present)
 To show available packages in the Nix expression `foo.nix`:
 
 ```console
-$ nix-env -f ./foo.nix -qa
+$ nix-env --file ./foo.nix --query --available
 foo-1.2.3
 ```
 
 To compare installed versions to what’s available:
 
 ```console
-$ nix-env -qc
+$ nix-env --query --compare-versions
 ...
 acrobat-reader-7.0 - ?      (package is not available at all)
 autoconf-2.59      = 2.59   (same version)
@@ -189,7 +189,7 @@ firefox-1.0.4      < 1.0.7  (a more recent version is available)
 To show all packages with “`zip`” in the name:
 
 ```console
-$ nix-env -qa '.*zip.*'
+$ nix-env --query --available '.*zip.*'
 bzip2-1.0.6
 gzip-1.6
 zip-3.0
@@ -199,7 +199,7 @@ zip-3.0
 To show all packages with “`firefox`” or “`chromium`” in the name:
 
 ```console
-$ nix-env -qa '.*(firefox|chromium).*'
+$ nix-env --query --available '.*(firefox|chromium).*'
 chromium-37.0.2062.94
 chromium-beta-38.0.2125.24
 firefox-32.0.3
@@ -210,6 +210,6 @@ firefox-with-plugins-13.0.1
 To show all packages in the latest revision of the Nixpkgs repository:
 
 ```console
-$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
+$ nix-env --file https://github.com/NixOS/nixpkgs/archive/master.tar.gz --query --available
 ```
 
diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md
index 63f0a0ff9..e04b22a91 100644
--- a/doc/manual/src/command-ref/nix-env/set-flag.md
+++ b/doc/manual/src/command-ref/nix-env/set-flag.md
@@ -46,16 +46,16 @@ To prevent the currently installed Firefox from being upgraded:
 $ nix-env --set-flag keep true firefox
 ```
 
-After this, `nix-env -u` will ignore Firefox.
+After this, `nix-env --upgrade ` will ignore Firefox.
 
 To disable the currently installed Firefox, then install a new Firefox
 while the old remains part of the profile:
 
 ```console
-$ nix-env -q
+$ nix-env --query
 firefox-2.0.0.9 (the current one)
 
-$ nix-env --preserve-installed -i firefox-2.0.0.11
+$ nix-env --preserve-installed --install firefox-2.0.0.11
 installing `firefox-2.0.0.11'
 building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
 collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
@@ -65,10 +65,10 @@ collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox'
 $ nix-env --set-flag active false firefox
 setting flag on `firefox-2.0.0.9'
 
-$ nix-env --preserve-installed -i firefox-2.0.0.11
+$ nix-env --preserve-installed --install firefox-2.0.0.11
 installing `firefox-2.0.0.11'
 
-$ nix-env -q
+$ nix-env --query
 firefox-2.0.0.11 (the enabled one)
 firefox-2.0.0.9 (the disabled one)
 ```
diff --git a/doc/manual/src/command-ref/nix-env/set.md b/doc/manual/src/command-ref/nix-env/set.md
index c1cf75739..b9950eeab 100644
--- a/doc/manual/src/command-ref/nix-env/set.md
+++ b/doc/manual/src/command-ref/nix-env/set.md
@@ -25,6 +25,6 @@ The following updates a profile such that its current generation will
 contain just Firefox:
 
 ```console
-$ nix-env -p /nix/var/nix/profiles/browser --set firefox
+$ nix-env --profile /nix/var/nix/profiles/browser --set firefox
 ```
 
diff --git a/doc/manual/src/command-ref/nix-env/switch-generation.md b/doc/manual/src/command-ref/nix-env/switch-generation.md
index e550325c4..38cf0534d 100644
--- a/doc/manual/src/command-ref/nix-env/switch-generation.md
+++ b/doc/manual/src/command-ref/nix-env/switch-generation.md
@@ -27,7 +27,7 @@ Switching will fail if the specified generation does not exist.
 # Examples
 
 ```console
-$ nix-env -G 42
+$ nix-env --switch-generation 42
 switching from generation 50 to 42
 ```
 
diff --git a/doc/manual/src/command-ref/nix-env/switch-profile.md b/doc/manual/src/command-ref/nix-env/switch-profile.md
index b389e4140..5ae2fdced 100644
--- a/doc/manual/src/command-ref/nix-env/switch-profile.md
+++ b/doc/manual/src/command-ref/nix-env/switch-profile.md
@@ -22,5 +22,5 @@ the symlink `~/.nix-profile` is made to point to *path*.
 # Examples
 
 ```console
-$ nix-env -S ~/my-profile
+$ nix-env --switch-profile ~/my-profile
 ```
diff --git a/doc/manual/src/command-ref/nix-env/uninstall.md b/doc/manual/src/command-ref/nix-env/uninstall.md
index e9ec8a15e..734cc7675 100644
--- a/doc/manual/src/command-ref/nix-env/uninstall.md
+++ b/doc/manual/src/command-ref/nix-env/uninstall.md
@@ -24,5 +24,5 @@ designated by the symbolic names *drvnames* are removed.
 
 ```console
 $ nix-env --uninstall gcc
-$ nix-env -e '.*' (remove everything)
+$ nix-env --uninstall '.*' (remove everything)
 ```
diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md
index f88ffcbee..322dfbda2 100644
--- a/doc/manual/src/command-ref/nix-env/upgrade.md
+++ b/doc/manual/src/command-ref/nix-env/upgrade.md
@@ -76,21 +76,21 @@ version is installed.
 # Examples
 
 ```console
-$ nix-env --upgrade -A nixpkgs.gcc
+$ nix-env --upgrade --attr nixpkgs.gcc
 upgrading `gcc-3.3.1' to `gcc-3.4'
 ```
 
 When there are no updates available, nothing will happen:
 
 ```console
-$ nix-env --upgrade -A nixpkgs.pan
+$ nix-env --upgrade --attr nixpkgs.pan
 ```
 
 Using `-A` is preferred when possible, as it is faster and unambiguous but
 it is also possible to upgrade to a specific version by matching the derivation name:
 
 ```console
-$ nix-env -u gcc-3.3.2 --always
+$ nix-env --upgrade gcc-3.3.2 --always
 upgrading `gcc-3.4' to `gcc-3.3.2'
 ```
 
@@ -98,7 +98,7 @@ To try to upgrade everything
 (matching packages based on the part of the derivation name without version):
 
 ```console
-$ nix-env -u
+$ nix-env --upgrade
 upgrading `hello-2.1.2' to `hello-2.1.3'
 upgrading `mozilla-1.2' to `mozilla-1.4'
 ```
diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index e55fb2afd..e1b4a3e80 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -88,7 +88,7 @@ Instantiate [store derivation]s from a Nix expression, and build them using `nix
 $ nix-instantiate test.nix (instantiate)
 /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
-$ nix-store -r $(nix-instantiate test.nix) (build)
+$ nix-store --realise $(nix-instantiate test.nix) (build)
 ...
 /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path)
 
@@ -100,30 +100,30 @@ dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
 You can also give a Nix expression on the command line:
 
 ```console
-$ nix-instantiate -E 'with import <nixpkgs> { }; hello'
+$ nix-instantiate --expr 'with import <nixpkgs> { }; hello'
 /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv
 ```
 
 This is equivalent to:
 
 ```console
-$ nix-instantiate '<nixpkgs>' -A hello
+$ nix-instantiate '<nixpkgs>' --attr hello
 ```
 
 Parsing and evaluating Nix expressions:
 
 ```console
-$ nix-instantiate --parse -E '1 + 2'
+$ nix-instantiate --parse --expr '1 + 2'
 1 + 2
 ```
 
 ```console
-$ nix-instantiate --eval -E '1 + 2'
+$ nix-instantiate --eval --expr '1 + 2'
 3
 ```
 
 ```console
-$ nix-instantiate --eval --xml -E '1 + 2'
+$ nix-instantiate --eval --xml --expr '1 + 2'
 <?xml version='1.0' encoding='utf-8'?>
 <expr>
   <int value="3" />
@@ -133,7 +133,7 @@ $ nix-instantiate --eval --xml -E '1 + 2'
 The difference between non-strict and strict evaluation:
 
 ```console
-$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }'
+$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }'
 ...
   <attr name="x">
     <string value="foo" />
@@ -148,7 +148,7 @@ Note that `y` is left unevaluated (the XML representation doesn’t
 attempt to show non-normal forms).
 
 ```console
-$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }'
+$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }'
 ...
   <attr name="x">
     <string value="foo" />
diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md
index 576e5ba0b..195b72be5 100644
--- a/doc/manual/src/command-ref/nix-shell.md
+++ b/doc/manual/src/command-ref/nix-shell.md
@@ -89,7 +89,7 @@ All options not listed here are passed to `nix-store
   - `--packages` / `-p` *packages*…\
     Set up an environment in which the specified packages are present.
     The command line arguments are interpreted as attribute names inside
-    the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk`
+    the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk`
     will start a shell in which the packages denoted by the attribute
     names `libjpeg` and `openjdk` are present.
 
@@ -118,7 +118,7 @@ To build the dependencies of the package Pan, and start an interactive
 shell in which to build it:
 
 ```console
-$ nix-shell '<nixpkgs>' -A pan
+$ nix-shell '<nixpkgs>' --attr pan
 [nix-shell]$ eval ${unpackPhase:-unpackPhase}
 [nix-shell]$ cd $sourceRoot
 [nix-shell]$ eval ${patchPhase:-patchPhase}
@@ -137,7 +137,7 @@ To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 
 ```console
-$ nix-shell '<nixpkgs>' -A pan --pure \
+$ nix-shell '<nixpkgs>' --attr pan --pure \
     --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
 ```
 
@@ -146,13 +146,13 @@ Nix expressions can also be given on the command line using the `-E` and
 packages `sqlite` and `libX11`:
 
 ```console
-$ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
+$ nix-shell --expr 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""'
 ```
 
 A shorter way to do the same is:
 
 ```console
-$ nix-shell -p sqlite xorg.libX11
+$ nix-shell --packages sqlite xorg.libX11
 [nix-shell]$ echo $NIX_LDFLAGS
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
 ```
@@ -162,7 +162,7 @@ the `buildInputs = [ ... ]` shown above, not only package names. So the
 following is also legal:
 
 ```console
-$ nix-shell -p sqlite 'git.override { withManual = false; }'
+$ nix-shell --packages sqlite 'git.override { withManual = false; }'
 ```
 
 The `-p` flag looks up Nixpkgs in the Nix search path. You can override
@@ -171,7 +171,7 @@ gives you a shell containing the Pan package from a specific revision of
 Nixpkgs:
 
 ```console
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell --packages pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -185,7 +185,7 @@ done by starting the script with the following lines:
 
 ```bash
 #! /usr/bin/env nix-shell
-#! nix-shell -i real-interpreter -p packages
+#! nix-shell -i real-interpreter --packages packages
 ```
 
 where *real-interpreter* is the “real” script interpreter that will be
@@ -202,7 +202,7 @@ For example, here is a Python script that depends on Python and the
 
 ```python
 #! /usr/bin/env nix-shell
-#! nix-shell -i python -p python pythonPackages.prettytable
+#! nix-shell -i python --packages python pythonPackages.prettytable
 
 import prettytable
 
@@ -217,7 +217,7 @@ requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages:
 
 ```perl
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
+#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
 
 use HTML::TokeParser::Simple;
 
@@ -235,7 +235,7 @@ package like Terraform:
 
 ```bash
 #! /usr/bin/env nix-shell
-#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
+#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])"
 
 terraform apply
 ```
@@ -251,7 +251,7 @@ branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
+#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
 #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
 import Network.Curl.Download
diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md
index 62656d599..c2f3c42ef 100644
--- a/doc/manual/src/command-ref/nix-store/dump.md
+++ b/doc/manual/src/command-ref/nix-store/dump.md
@@ -23,7 +23,7 @@ produce the same NAR archive. For instance, directory entries are
 always sorted so that the actual on-disk order doesn’t influence the
 result.  This means that the cryptographic hash of a NAR dump of a
 path is usable as a fingerprint of the contents of the path. Indeed,
-the hashes of store paths stored in Nix’s database (see `nix-store -q
+the hashes of store paths stored in Nix’s database (see `nix-store --query
 --hash`) are SHA-256 hashes of the NAR dump of each store path.
 
 NAR archives support filenames of unlimited length and 64-bit file
diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md
index aeea38636..1bc46f53b 100644
--- a/doc/manual/src/command-ref/nix-store/export.md
+++ b/doc/manual/src/command-ref/nix-store/export.md
@@ -31,7 +31,7 @@ To copy a whole closure, do something
 like:
 
 ```console
-$ nix-store --export $(nix-store -qR paths) > out
+$ nix-store --export $(nix-store --query --requisites paths) > out
 ```
 
 To import the whole closure again, run:
diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md
index bf6566555..dd9a6bf21 100644
--- a/doc/manual/src/command-ref/nix-store/opt-common.md
+++ b/doc/manual/src/command-ref/nix-store/opt-common.md
@@ -11,7 +11,7 @@ The following options are allowed for all `nix-store` operations, but may not al
   be created in `/nix/var/nix/gcroots/auto/`. For instance,
 
   ```console
-  $ nix-store --add-root /home/eelco/bla/result -r ...
+  $ nix-store --add-root /home/eelco/bla/result --realise ...
 
   $ ls -l /nix/var/nix/gcroots/auto
   lrwxrwxrwx    1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result
diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md
index 9f7dbd3e8..cd45a4932 100644
--- a/doc/manual/src/command-ref/nix-store/query.md
+++ b/doc/manual/src/command-ref/nix-store/query.md
@@ -145,7 +145,7 @@ Print the closure (runtime dependencies) of the `svn` program in the
 current user environment:
 
 ```console
-$ nix-store -qR $(which svn)
+$ nix-store --query --requisites $(which svn)
 /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
 /nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
 ...
@@ -154,7 +154,7 @@ $ nix-store -qR $(which svn)
 Print the build-time dependencies of `svn`:
 
 ```console
-$ nix-store -qR $(nix-store -qd $(which svn))
+$ nix-store --query --requisites $(nix-store --query --deriver $(which svn))
 /nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
 /nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
 /nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
@@ -168,7 +168,7 @@ the derivation (`-qd`), not the closure of the output path that contains
 Show the build-time dependencies as a tree:
 
 ```console
-$ nix-store -q --tree $(nix-store -qd $(which svn))
+$ nix-store --query --tree $(nix-store --query --deriver $(which svn))
 /nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
 +---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
 +---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
@@ -180,7 +180,7 @@ $ nix-store -q --tree $(nix-store -qd $(which svn))
 Show all paths that depend on the same OpenSSL library as `svn`:
 
 ```console
-$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
+$ nix-store --query --referrers $(nix-store --query --binding openssl $(nix-store --query --deriver $(which svn)))
 /nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
 /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
 /nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
@@ -191,7 +191,7 @@ Show all paths that directly or indirectly depend on the Glibc (C
 library) used by `svn`:
 
 ```console
-$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
+$ nix-store --query --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
 /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
 /nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
 ...
@@ -204,7 +204,7 @@ Make a picture of the runtime dependency graph of the current user
 environment:
 
 ```console
-$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps
+$ nix-store --query --graph ~/.nix-profile | dot -Tps > graph.ps
 $ gv graph.ps
 ```
 
@@ -212,7 +212,7 @@ Show every garbage collector root that points to a store path that
 depends on `svn`:
 
 ```console
-$ nix-store -q --roots $(which svn)
+$ nix-store --query --roots $(which svn)
 /nix/var/nix/profiles/default-81-link
 /nix/var/nix/profiles/default-82-link
 /home/eelco/.local/state/nix/profiles/profile-97-link
diff --git a/doc/manual/src/command-ref/nix-store/read-log.md b/doc/manual/src/command-ref/nix-store/read-log.md
index 4a88e9382..d1ff17891 100644
--- a/doc/manual/src/command-ref/nix-store/read-log.md
+++ b/doc/manual/src/command-ref/nix-store/read-log.md
@@ -27,7 +27,7 @@ substitute, then the log is unavailable.
 # Example
 
 ```console
-$ nix-store -l $(which ktorrent)
+$ nix-store --read-log $(which ktorrent)
 building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1
 unpacking sources
 unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz
diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md
index f61a20100..6b50d2145 100644
--- a/doc/manual/src/command-ref/nix-store/realise.md
+++ b/doc/manual/src/command-ref/nix-store/realise.md
@@ -99,7 +99,7 @@ This operation is typically used to build [store derivation]s produced by
 [store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 ```console
-$ nix-store -r $(nix-instantiate ./test.nix)
+$ nix-store --realise $(nix-instantiate ./test.nix)
 /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
 ```
 
@@ -108,7 +108,7 @@ This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does.
 To test whether a previously-built derivation is deterministic:
 
 ```console
-$ nix-build '<nixpkgs>' -A hello --check -K
+$ nix-build '<nixpkgs>' --attr hello --check -K
 ```
 
 Use [`nix-store --read-log`](./read-log.md) to show the stderr and stdout of a build:
diff --git a/doc/manual/src/command-ref/nix-store/verify-path.md b/doc/manual/src/command-ref/nix-store/verify-path.md
index 59ffe92a3..927201599 100644
--- a/doc/manual/src/command-ref/nix-store/verify-path.md
+++ b/doc/manual/src/command-ref/nix-store/verify-path.md
@@ -24,6 +24,6 @@ path has changed, and 1 otherwise.
 To verify the integrity of the `svn` command and all its dependencies:
 
 ```console
-$ nix-store --verify-path $(nix-store -qR $(which svn))
+$ nix-store --verify-path $(nix-store --query --requisites $(which svn))
 ```
 
diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md
index 7a012250d..54c0a1d0d 100644
--- a/doc/manual/src/command-ref/opt-common.md
+++ b/doc/manual/src/command-ref/opt-common.md
@@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options:
     }: ...
     ```
 
-    So if you call this Nix expression (e.g., when you do `nix-env -iA
+    So if you call this Nix expression (e.g., when you do `nix-env --install --attr
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](@docroot@/language/builtins.md) for
     the `system` argument. You can override this using `--arg`, e.g.,
-    `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that
+    `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 
@@ -199,7 +199,7 @@ Most Nix commands accept the following command-line options:
     For `nix-shell`, this option is commonly used to give you a shell in
     which you can build the packages returned by the expression. If you
     want to get a shell which contain the *built* packages ready for
-    use, give your expression to the `nix-shell -p` convenience flag
+    use, give your expression to the `nix-shell --packages ` convenience flag
     instead.
 
   - <span id="opt-I">[`-I`](#opt-I)</span> *path*\
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index ca69f076a..b954a2167 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -77,7 +77,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell -A devShells.x86_64-linux.native-clang11StdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
 ```
 
 > **Note**
@@ -139,7 +139,7 @@ $ nix build .#packages.aarch64-linux.default
 for flake-enabled Nix, or
 
 ```console
-$ nix-build -A packages.aarch64-linux.default
+$ nix-build --attr packages.aarch64-linux.default
 ```
 
 for classic Nix.
@@ -166,7 +166,7 @@ $ nix build .#nix-ccacheStdenv
 for flake-enabled Nix, or
 
 ```console
-$ nix-build -A nix-ccacheStdenv
+$ nix-build --attr nix-ccacheStdenv
 ```
 
 for classic Nix.
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index eeb19ad50..97465d2e1 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -163,7 +163,7 @@
     build-time dependencies, while the closure of its output path is
     equivalent to its runtime dependencies. For correct deployment it
     is necessary to deploy whole closures, since otherwise at runtime
-    files could be missing. The command `nix-store -qR` prints out
+    files could be missing. The command `nix-store --query --requisites ` prints out
     closures of store paths.
 
     As an example, if the [store object] at path `P` contains a [reference]
diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md
index 24efc4681..6d09f54d8 100644
--- a/doc/manual/src/installation/upgrading.md
+++ b/doc/manual/src/installation/upgrading.md
@@ -2,13 +2,13 @@
 
 Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
 'nix-channel --update &&
-nix-env -iA nixpkgs.nix &&
+nix-env --install --attr nixpkgs.nix &&
 launchctl remove org.nixos.nix-daemon &&
 launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'`
 
 Single-user installations of Nix should run this: `nix-channel --update;
-nix-env -iA nixpkgs.nix nixpkgs.cacert`
+nix-env --install --attr nixpkgs.nix nixpkgs.cacert`
 
 Multi-user Nix users on Linux should run this with sudo: `nix-channel
---update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl
+--update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl
 daemon-reload; systemctl restart nix-daemon`
diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md
index b54346db8..76489bc1b 100644
--- a/doc/manual/src/introduction.md
+++ b/doc/manual/src/introduction.md
@@ -76,7 +76,7 @@ there after an upgrade.  This means that you can _roll back_ to the
 old version:
 
 ```console
-$ nix-env --upgrade -A nixpkgs.some-package
+$ nix-env --upgrade --attr nixpkgs.some-package
 $ nix-env --rollback
 ```
 
@@ -122,7 +122,7 @@ Nix expressions generally describe how to build a package from
 source, so an installation action like
 
 ```console
-$ nix-env --install -A nixpkgs.firefox
+$ nix-env --install --attr nixpkgs.firefox
 ```
 
 _could_ cause quite a bit of build activity, as not only Firefox but
@@ -158,7 +158,7 @@ Pan newsreader, as described by [its
 Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix):
 
 ```console
-$ nix-shell '<nixpkgs>' -A pan
+$ nix-shell '<nixpkgs>' --attr pan
 ```
 
 You’re then dropped into a shell where you can edit, build and test
diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md
index 5f1d7a89c..6b86e763e 100644
--- a/doc/manual/src/package-management/basic-package-mgmt.md
+++ b/doc/manual/src/package-management/basic-package-mgmt.md
@@ -47,7 +47,7 @@ $ nix-channel --update
 You can view the set of available packages in Nixpkgs:
 
 ```console
-$ nix-env -qaP
+$ nix-env --query --available --attr-path
 nixpkgs.aterm                       aterm-2.2
 nixpkgs.bash                        bash-3.0
 nixpkgs.binutils                    binutils-2.15
@@ -65,7 +65,7 @@ If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
 then you need to pass the path to your Nixpkgs tree using the `-f` flag:
 
 ```console
-$ nix-env -qaPf /path/to/nixpkgs
+$ nix-env --query --available --attr-path --file /path/to/nixpkgs
 aterm                               aterm-2.2
 bash                                bash-3.0
 …
@@ -77,7 +77,7 @@ Nixpkgs.
 You can filter the packages by name:
 
 ```console
-$ nix-env -qaP firefox
+$ nix-env --query --available --attr-path firefox
 nixpkgs.firefox-esr                 firefox-91.3.0esr
 nixpkgs.firefox                     firefox-94.0.1
 ```
@@ -85,7 +85,7 @@ nixpkgs.firefox                     firefox-94.0.1
 and using regular expressions:
 
 ```console
-$ nix-env -qaP 'firefox.*'
+$ nix-env --query --available --attr-path 'firefox.*'
 ```
 
 It is also possible to see the *status* of available packages, i.e.,
@@ -93,7 +93,7 @@ whether they are installed into the user environment and/or present in
 the system:
 
 ```console
-$ nix-env -qaPs
+$ nix-env --query --available --attr-path --status
 …
 -PS  nixpkgs.bash                bash-3.0
 --S  nixpkgs.binutils            binutils-2.15
@@ -110,10 +110,10 @@ which is Nix’s mechanism for doing binary deployment. It just means that
 Nix knows that it can fetch a pre-built package from somewhere
 (typically a network server) instead of building it locally.
 
-You can install a package using `nix-env -iA`. For instance,
+You can install a package using `nix-env --install --attr `. For instance,
 
 ```console
-$ nix-env -iA nixpkgs.subversion
+$ nix-env --install --attr nixpkgs.subversion
 ```
 
 will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
@@ -143,14 +143,14 @@ instead of the attribute path, as `nix-env` does not record which attribute
 was used for installing:
 
 ```console
-$ nix-env -e subversion
+$ nix-env --uninstall subversion
 ```
 
 Upgrading to a new version is just as easy. If you have a new release of
 Nix Packages, you can do:
 
 ```console
-$ nix-env -uA nixpkgs.subversion
+$ nix-env --upgrade --attr nixpkgs.subversion
 ```
 
 This will *only* upgrade Subversion if there is a “newer” version in the
@@ -163,15 +163,15 @@ whatever version is in the Nix expressions, use `-i` instead of `-u`;
 You can also upgrade all packages for which there are newer versions:
 
 ```console
-$ nix-env -u
+$ nix-env --upgrade
 ```
 
 Sometimes it’s useful to be able to ask what `nix-env` would do, without
 actually doing it. For instance, to find out what packages would be
-upgraded by `nix-env -u`, you can do
+upgraded by `nix-env --upgrade `, you can do
 
 ```console
-$ nix-env -u --dry-run
+$ nix-env --upgrade --dry-run
 (dry run; not doing anything)
 upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
 upgrading `graphviz-1.10' to `graphviz-1.12'
diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md
index 5befad9f8..855eaf470 100644
--- a/doc/manual/src/package-management/binary-cache-substituter.md
+++ b/doc/manual/src/package-management/binary-cache-substituter.md
@@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is
 not part of the Nix distribution, but you can install it from Nixpkgs:
 
 ```console
-$ nix-env -iA nixpkgs.nix-serve
+$ nix-env --install --attr nixpkgs.nix-serve
 ```
 
 You can then start the server, listening for HTTP connections on
@@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using
 `--substituters`, e.g.:
 
 ```console
-$ nix-env -iA nixpkgs.firefox --substituters http://avalon:8080/
+$ nix-env --install --attr nixpkgs.firefox --substituters http://avalon:8080/
 ```
 
 The option `substituters` tells Nix to use this binary cache in
diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md
index 93c8b41a6..8e4da180b 100644
--- a/doc/manual/src/package-management/channels.md
+++ b/doc/manual/src/package-management/channels.md
@@ -43,7 +43,7 @@ operations (via the symlink `~/.nix-defexpr/channels`). Consequently,
 you can then say
 
 ```console
-$ nix-env -u
+$ nix-env --upgrade
 ```
 
 to upgrade all packages in your profile to the latest versions available
diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md
index d3fac4d76..14326298b 100644
--- a/doc/manual/src/package-management/copy-closure.md
+++ b/doc/manual/src/package-management/copy-closure.md
@@ -15,7 +15,7 @@ With `nix-store
 path (that is, the path and all its dependencies) to a file, and then
 unpack that file into another Nix store. For example,
 
-    $ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure
+    $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure
 
 writes the closure of Firefox to a file. You can then copy this file to
 another machine and install the closure:
@@ -27,7 +27,7 @@ store are ignored. It is also possible to pipe the export into another
 command, e.g. to copy and install a closure directly to/on another
 machine:
 
-    $ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \
+    $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \
         ssh alice@itchy.example.org "bunzip2 | nix-store --import"
 
 However, `nix-copy-closure` is generally more efficient because it only
diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md
index d1a2580d4..1d9e672a8 100644
--- a/doc/manual/src/package-management/profiles.md
+++ b/doc/manual/src/package-management/profiles.md
@@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
 would be what we would obtain if we had done
 
 ```console
-$ nix-env -iA nixpkgs.subversion
+$ nix-env --install --attr nixpkgs.subversion
 ```
 
 on a set of Nix expressions that contained Subversion 1.1.2.
@@ -54,7 +54,7 @@ environment is generated based on the current one. For instance,
 generation 43 was created from generation 42 when we did
 
 ```console
-$ nix-env -iA nixpkgs.subversion nixpkgs.firefox
+$ nix-env --install --attr nixpkgs.subversion nixpkgs.firefox
 ```
 
 on a set of Nix expressions that contained Firefox and a new version of
@@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by
 (abbreviation `-p`):
 
 ```console
-$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion
+$ nix-env --profile /nix/var/nix/profiles/other-profile --install --attr nixpkgs.subversion
 ```
 
 This will *not* change the `~/.nix-profile` symlink.
diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md
index c59933f61..7014c3cc8 100644
--- a/doc/manual/src/package-management/ssh-substituter.md
+++ b/doc/manual/src/package-management/ssh-substituter.md
@@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are
 available on the server `avalon`:
 
 ```console
-$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon
+$ nix-env --install --attr nixpkgs.firefox --substituters ssh://alice@avalon
 ```
 
 This works similar to the binary cache substituter that Nix usually
@@ -25,7 +25,7 @@ You can also copy the closure of some store path, without installing it
 into your profile, e.g.
 
 ```console
-$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters
+$ nix-store --realise /nix/store/m85bxg…-firefox-34.0.5 --substituters
 ssh://alice@avalon
 ```
 

From afd9bd787dddc4e0d3e370f1b043e442ce1a9dbc Mon Sep 17 00:00:00 2001
From: Alexander Schmolck <a.schmolck+git@gmail.com>
Date: Sun, 30 Apr 2023 14:55:08 +0100
Subject: [PATCH 014/183] Also use long options in src/nix/*.md

---
 src/nix/build.md       | 2 +-
 src/nix/copy.md        | 2 +-
 src/nix/develop.md     | 2 +-
 src/nix/eval.md        | 2 +-
 src/nix/flake-check.md | 2 +-
 src/nix/nar-ls.md      | 4 ++--
 src/nix/nix.md         | 4 ++--
 src/nix/path-info.md   | 8 ++++----
 src/nix/search.md      | 4 ++--
 src/nix/shell.md       | 8 ++++----
 src/nix/store-ls.md    | 4 ++--
 src/nix/upgrade-nix.md | 2 +-
 src/nix/verify.md      | 2 +-
 13 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/nix/build.md b/src/nix/build.md
index ee414dc86..0fbb39cc3 100644
--- a/src/nix/build.md
+++ b/src/nix/build.md
@@ -44,7 +44,7 @@ R""(
   `release.nix`:
 
   ```console
-  # nix build -f release.nix build.x86_64-linux
+  # nix build --file release.nix build.x86_64-linux
   ```
 
 * Build a NixOS system configuration from a flake, and make a profile
diff --git a/src/nix/copy.md b/src/nix/copy.md
index 25e0ddadc..199006436 100644
--- a/src/nix/copy.md
+++ b/src/nix/copy.md
@@ -15,7 +15,7 @@ R""(
   SSH:
 
   ```console
-  # nix copy -s --to ssh://server /run/current-system
+  # nix copy --substitute-on-destination --to ssh://server /run/current-system
   ```
 
   The `-s` flag causes the remote machine to try to substitute missing
diff --git a/src/nix/develop.md b/src/nix/develop.md
index c49b39669..1b5a8aeba 100644
--- a/src/nix/develop.md
+++ b/src/nix/develop.md
@@ -69,7 +69,7 @@ R""(
 * Run a series of script commands:
 
   ```console
-  # nix develop --command bash -c "mkdir build && cmake .. && make"
+  # nix develop --command bash --command "mkdir build && cmake .. && make"
   ```
 
 # Description
diff --git a/src/nix/eval.md b/src/nix/eval.md
index 3b510737a..48d5aa597 100644
--- a/src/nix/eval.md
+++ b/src/nix/eval.md
@@ -18,7 +18,7 @@ R""(
 * Evaluate a Nix expression from a file:
 
   ```console
-  # nix eval -f ./my-nixpkgs hello.name
+  # nix eval --file ./my-nixpkgs hello.name
   ```
 
 * Get the current version of the `nixpkgs` flake:
diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md
index 07031c909..c8307f8d8 100644
--- a/src/nix/flake-check.md
+++ b/src/nix/flake-check.md
@@ -68,6 +68,6 @@ The following flake output attributes must be
 In addition, the `hydraJobs` output is evaluated in the same way as
 Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested
 attribute set of derivations). Similarly, the
-`legacyPackages`.*system* output is evaluated like `nix-env -qa`.
+`legacyPackages`.*system* output is evaluated like `nix-env --query --available `.
 
 )""
diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md
index d373f9715..5a03c5d82 100644
--- a/src/nix/nar-ls.md
+++ b/src/nix/nar-ls.md
@@ -5,7 +5,7 @@ R""(
 * To list a specific file in a NAR:
 
   ```console
-  # nix nar ls -l ./hello.nar /bin/hello
+  # nix nar ls --long ./hello.nar /bin/hello
   -r-xr-xr-x                38184 hello
   ```
 
@@ -13,7 +13,7 @@ R""(
   format:
 
   ```console
-  # nix nar ls --json -R ./hello.nar /bin
+  # nix nar ls --json --recursive ./hello.nar /bin
   {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}}
   ```
 
diff --git a/src/nix/nix.md b/src/nix/nix.md
index 1ef6c7fcd..8a850ae83 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -197,7 +197,7 @@ operate are determined as follows:
   of all outputs of the `glibc` package in the binary cache:
 
   ```console
-  # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
+  # nix path-info --closure-size --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
   /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123                 33208200
   /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin             36142896
   /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug          155787312
@@ -208,7 +208,7 @@ operate are determined as follows:
   and likewise, using a store path to a "drv" file to specify the derivation:
 
   ```console
-  # nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*'
+  # nix path-info --closure-size '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*'
   …
   ```
 * If you didn't specify the desired outputs, but the derivation has an
diff --git a/src/nix/path-info.md b/src/nix/path-info.md
index 6ad23a02e..2dda866d0 100644
--- a/src/nix/path-info.md
+++ b/src/nix/path-info.md
@@ -13,7 +13,7 @@ R""(
   closure, sorted by size:
 
   ```console
-  # nix path-info -rS /run/current-system | sort -nk2
+  # nix path-info --recursive --closure-size /run/current-system | sort -nk2
   /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty                                                96
   /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers                                         112
   …
@@ -25,7 +25,7 @@ R""(
   readable sizes:
 
   ```console
-  # nix path-info -rsSh nixpkgs#rustc
+  # nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc
   /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev      386.7K   69.1M
   /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0          5.9M   37.4M
   …
@@ -34,7 +34,7 @@ R""(
 * Check the existence of a path in a binary cache:
 
   ```console
-  # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
+  # nix path-info --recursive /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
   path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid
 
   ```
@@ -57,7 +57,7 @@ R""(
   size:
 
   ```console
-  # nix path-info --json --all -S \
+  # nix path-info --json --all --closure-size \
     | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'
   [
     …,
diff --git a/src/nix/search.md b/src/nix/search.md
index 4caa90654..0c5d22549 100644
--- a/src/nix/search.md
+++ b/src/nix/search.md
@@ -52,12 +52,12 @@ R""(
 * Search for packages containing `neovim` but hide ones containing either `gui` or `python`:
 
   ```console
-  # nix search nixpkgs neovim -e 'python|gui'
+  # nix search nixpkgs neovim --exclude 'python|gui'
   ```
   or
 
   ```console
-  # nix search nixpkgs neovim -e 'python' -e 'gui'
+  # nix search nixpkgs neovim --exclude 'python' --exclude 'gui'
   ```
 
 # Description
diff --git a/src/nix/shell.md b/src/nix/shell.md
index 13a389103..1668104b1 100644
--- a/src/nix/shell.md
+++ b/src/nix/shell.md
@@ -19,26 +19,26 @@ R""(
 * Run GNU Hello:
 
   ```console
-  # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'
+  # nix shell nixpkgs#hello --command hello --greeting 'Hi everybody!'
   Hi everybody!
   ```
 
 * Run multiple commands in a shell environment:
 
   ```console
-  # nix shell nixpkgs#gnumake -c sh -c "cd src && make"
+  # nix shell nixpkgs#gnumake --command sh --command "cd src && make"
   ```
 
 * Run GNU Hello in a chroot store:
 
   ```console
-  # nix shell --store ~/my-nix nixpkgs#hello -c hello
+  # nix shell --store ~/my-nix nixpkgs#hello --command hello
   ```
 
 * Start a shell providing GNU Hello in a chroot store:
 
   ```console
-  # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash
+  # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive --command bash
   ```
 
   Note that it's necessary to specify `bash` explicitly because your
diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md
index 836efce42..14c4627c9 100644
--- a/src/nix/store-ls.md
+++ b/src/nix/store-ls.md
@@ -5,7 +5,7 @@ R""(
 * To list the contents of a store path in a binary cache:
 
   ```console
-  # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+  # nix store ls --store https://cache.nixos.org/ --long --recursive /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
   dr-xr-xr-x                    0 ./bin
   -r-xr-xr-x                38184 ./bin/hello
   dr-xr-xr-x                    0 ./share
@@ -15,7 +15,7 @@ R""(
 * To show information about a specific file in a binary cache:
 
   ```console
-  # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
+  # nix store ls --store https://cache.nixos.org/ --long /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
   -r-xr-xr-x                38184 hello
   ```
 
diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md
index 08757aebd..cce88c397 100644
--- a/src/nix/upgrade-nix.md
+++ b/src/nix/upgrade-nix.md
@@ -11,7 +11,7 @@ R""(
 * Upgrade Nix in a specific profile:
 
   ```console
-  # nix upgrade-nix -p ~alice/.local/state/nix/profiles/profile
+  # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile
   ```
 
 # Description
diff --git a/src/nix/verify.md b/src/nix/verify.md
index cc1122c02..e1d55eab4 100644
--- a/src/nix/verify.md
+++ b/src/nix/verify.md
@@ -12,7 +12,7 @@ R""(
   signatures:
 
   ```console
-  # nix store verify -r -n2 --no-contents $(type -p firefox)
+  # nix store verify --recursive --sigs-needed 2 --no-contents $(type -p firefox)
   ```
 
 * Verify a store path in the binary cache `https://cache.nixos.org/`:

From 992e2ed0cf5d1e15eab1c6c523403a7fb7d3136a Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Wed, 17 May 2023 11:34:45 +0400
Subject: [PATCH 015/183] Add a test for ControlMaster

---
 tests/nixos/nix-copy.nix | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix
index ee8b77100..16c477bf9 100644
--- a/tests/nixos/nix-copy.nix
+++ b/tests/nixos/nix-copy.nix
@@ -23,6 +23,12 @@ in {
           nix.settings.substituters = lib.mkForce [ ];
           nix.settings.experimental-features = [ "nix-command" ];
           services.getty.autologinUser = "root";
+          programs.ssh.extraConfig = ''
+            Host *
+                ControlMaster auto
+                ControlPath ~/.ssh/master-%h:%r@%n:%p
+                ControlPersist 15m
+          '';
         };
 
       server =
@@ -62,6 +68,10 @@ in {
     client.wait_for_text("done")
     server.succeed("nix-store --check-validity ${pkgA}")
 
+    # Check that ControlMaster is working
+    client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n")
+    client.wait_for_text("done")
+
     client.copy_from_host("key", "/root/.ssh/id_ed25519")
     client.succeed("chmod 600 /root/.ssh/id_ed25519")
 

From d8bfeda1643f434ae015caa4f59d9f4b7f4270d7 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 11 May 2023 19:23:13 +0200
Subject: [PATCH 016/183] document identifier syntax for attribute sets

it's more likely for readers to find it right there.

this also slightly rewords examples to make them stand out better.
in the long run there probably needs to be a dedicated section on formal syntax, and better highlighting of examples.
---
 doc/manual/src/language/constructs.md | 47 ++++++++++++++++++---------
 doc/manual/src/language/operators.md  | 11 +++----
 doc/manual/src/language/values.md     | 20 +++++++++---
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md
index 1c01f2cc7..c53eb8889 100644
--- a/doc/manual/src/language/constructs.md
+++ b/doc/manual/src/language/constructs.md
@@ -2,8 +2,11 @@
 
 ## Recursive sets
 
-Recursive sets are just normal sets, but the attributes can refer to
-each other. For example,
+Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other.
+
+> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}`
+
+Example:
 
 ```nix
 rec {
@@ -12,7 +15,9 @@ rec {
 }.x
 ```
 
-evaluates to `123`. Note that without `rec` the binding `x = y;` would
+This evaluates to `123`.
+
+Note that without `rec` the binding `x = y;` would
 refer to the variable `y` in the surrounding scope, if one exists, and
 would be invalid if no such variable exists. That is, in a normal
 (non-recursive) set, attributes are not added to the lexical scope; in a
@@ -33,7 +38,10 @@ will crash with an `infinite recursion encountered` error message.
 ## Let-expressions
 
 A let-expression allows you to define local variables for an expression.
-For instance,
+
+> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr*
+
+Example:
 
 ```nix
 let
@@ -42,18 +50,19 @@ let
 in x + y
 ```
 
-evaluates to `"foobar"`.
+This evaluates to `"foobar"`.
 
 ## Inheriting attributes
 
-When defining a set or in a let-expression it is often convenient to
-copy variables from the surrounding lexical scope (e.g., when you want
-to propagate attributes). This can be shortened using the `inherit`
-keyword. For instance,
+When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes).
+This can be shortened using the `inherit` keyword.
+
+Example:
 
 ```nix
 let x = 123; in
-{ inherit x;
+{
+  inherit x;
   y = 456;
 }
 ```
@@ -62,15 +71,23 @@ is equivalent to
 
 ```nix
 let x = 123; in
-{ x = x;
+{
+  x = x;
   y = 456;
 }
 ```
 
-and both evaluate to `{ x = 123; y = 456; }`. (Note that this works
-because `x` is added to the lexical scope by the `let` construct.) It is
-also possible to inherit attributes from another set. For instance, in
-this fragment from `all-packages.nix`,
+and both evaluate to `{ x = 123; y = 456; }`.
+
+> **Note**
+>
+> This works because `x` is added to the lexical scope by the `let` construct.
+
+It is also possible to inherit attributes from another attribute set.
+
+Example:
+
+In this fragment from `all-packages.nix`,
 
 ```nix
 graphviz = (import ../tools/graphics/graphviz) {
diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md
index 3e929724d..f8382ae19 100644
--- a/doc/manual/src/language/operators.md
+++ b/doc/manual/src/language/operators.md
@@ -35,17 +35,14 @@
 
 ## Attribute selection
 
+> *attrset* `.` *attrpath* \[ `or` *expr* \]
+
 Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*.
 If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation.
 
-<!-- FIXME: the following should to into its own language syntax section, but that needs more work to fit in well -->
+An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set).
 
-An attribute path is a dot-separated list of attribute names.
-An attribute name can be an identifier or a string.
-
-> *attrpath* = *name* [ `.` *name* ]... \
-> *name* = *identifier* | *string* \
-> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*`
+> *attrpath* = *name* [ `.` *name* ]...
 
 [Attribute selection]: #attribute-selection
 
diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md
index 9d0301753..2ae3e143a 100644
--- a/doc/manual/src/language/values.md
+++ b/doc/manual/src/language/values.md
@@ -164,9 +164,17 @@ Note that lists are only lazy in values, and they are strict in length.
 
 An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`).
 
+An attribute name can be an identifier or a [string](#string).
+An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`).
+
+> *name* = *identifier* | *string* \
+> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*`
+
 Names and values are separated by an equal sign (`=`).
 Each value is an arbitrary expression terminated by a semicolon (`;`).
 
+> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}`
+
 Attributes can appear in any order.
 An attribute name may only occur once.
 
@@ -182,15 +190,19 @@ Example:
 
 This defines a set with attributes named `x`, `text`, `y`.
 
-Attributes can be selected from a set using the `.` operator. For
-instance,
+Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection).
+
+Example:
 
 ```nix
 { a = "Foo"; b = "Bar"; }.a
 ```
 
-evaluates to `"Foo"`. It is possible to provide a default value in an
-attribute selection using the `or` keyword:
+This evaluates to `"Foo"`.
+
+It is possible to provide a default value in an attribute selection using the `or` keyword.
+
+Example:
 
 ```nix
 { a = "Foo"; b = "Bar"; }.c or "Xyzzy"

From 549f8dc97b56babb0e23e2c097dce2fa283b6d09 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 11 May 2023 20:04:16 +0200
Subject: [PATCH 017/183] document `builtins.currentTime`

---
 doc/manual/src/language/builtin-constants.md | 23 ++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md
index c6bc9b74c..204daab6e 100644
--- a/doc/manual/src/language/builtin-constants.md
+++ b/doc/manual/src/language/builtin-constants.md
@@ -17,3 +17,26 @@ These constants are built into the Nix language evaluator:
   The built-in value `currentSystem` evaluates to the Nix platform
   identifier for the Nix installation on which the expression is being
   evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
+
+- [`builtins.currentTime`]{#builtins-currentTime} (integer)
+
+  Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
+  Repeated references to that name will re-use the initially obtained value.
+
+  Example:
+
+  ```console
+  $ nix repl
+  Welcome to Nix 2.15.1 Type :? for help.
+
+  nix-repl> builtins.currentTime
+  1683705525
+
+  nix-repl> builtins.currentTime
+  1683705525
+  ```
+
+  > **Warning**
+  >
+  > This is intended for testing and debugging only.
+  > Do not use it in production systems, since an impure, that is, non-constant value will break reproducibility.

From 6a5a8f51bb9773ab4aeac8f508872bcf5ec6fda2 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 17 May 2023 14:59:47 +0200
Subject: [PATCH 018/183] add cross-references to pure evaluation mode

use consistent wording everywhere.
add some details on the configuration option documentation.
---
 doc/manual/src/language/builtin-constants.md |  9 +++++----
 src/libexpr/eval.hh                          |  8 +++++++-
 src/libexpr/primops.cc                       |  2 +-
 src/libexpr/primops/fetchTree.cc             | 18 ++++++------------
 src/libutil/experimental-features.cc         |  2 ++
 5 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md
index 204daab6e..e6bc7e915 100644
--- a/doc/manual/src/language/builtin-constants.md
+++ b/doc/manual/src/language/builtin-constants.md
@@ -18,6 +18,8 @@ These constants are built into the Nix language evaluator:
   identifier for the Nix installation on which the expression is being
   evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
 
+  Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+
 - [`builtins.currentTime`]{#builtins-currentTime} (integer)
 
   Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
@@ -36,7 +38,6 @@ These constants are built into the Nix language evaluator:
   1683705525
   ```
 
-  > **Warning**
-  >
-  > This is intended for testing and debugging only.
-  > Do not use it in production systems, since an impure, that is, non-constant value will break reproducibility.
+  The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation.
+
+  Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index a90ff34c0..ef8c3e39b 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -745,7 +745,13 @@ struct EvalSettings : Config
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
-        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+        R"(
+          Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
+
+          - Restrict file system and network access to files specified by cryptographic hash
+          - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
+        )"
+        };
 
     Setting<bool> enableImportFromDerivation{
         this, true, "allow-import-from-derivation",
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6fbd66389..7f36b021d 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1502,7 +1502,7 @@ static RegisterPrimOp primop_storePath({
       causes the path to be *copied* again to the Nix store, resulting
       in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
 
-      This function is not available in pure evaluation mode.
+      Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
     )",
     .fun = prim_storePath,
 });
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index cd7039025..fe880aaa8 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -286,9 +286,9 @@ static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
     .args = {"url"},
     .doc = R"(
-      Download the specified URL and return the path of the downloaded
-      file. This function is not available if [restricted evaluation
-      mode](../command-ref/conf-file.md) is enabled.
+      Download the specified URL and return the path of the downloaded file.
+
+      Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
     .fun = prim_fetchurl,
 });
@@ -338,8 +338,7 @@ static RegisterPrimOp primop_fetchTarball({
       stdenv.mkDerivation { … }
       ```
 
-      This function is not available if [restricted evaluation
-      mode](../command-ref/conf-file.md) is enabled.
+      Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
     .fun = prim_fetchTarball,
 });
@@ -470,14 +469,9 @@ static RegisterPrimOp primop_fetchGit({
           }
           ```
 
-          > **Note**
-          >
-          > Nix will refetch the branch in accordance with
-          > the option `tarball-ttl`.
+          Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting.
 
-          > **Note**
-          >
-          > This behavior is disabled in *Pure evaluation mode*.
+          This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
 
         - To fetch the content of a checked-out work directory:
 
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index ad0ec0427..9bebf77e5 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -50,6 +50,8 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
             or other impure derivations can rely on impure derivations. Finally,
             an impure derivation cannot also be
             [content-addressed](#xp-feature-ca-derivations).
+
+            This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime).
         )",
     },
     {

From 76f40c02bfac1de0fa2e31003491b48504fedc65 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 17 May 2023 15:03:27 +0200
Subject: [PATCH 019/183] fix wording on output-addressed store objects

hashing is an implementation detail.
add references to the other terms.
---
 doc/manual/src/glossary.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index eeb19ad50..5e9103cab 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -101,11 +101,8 @@
     derivation.
 
   - [output-addressed store object]{#gloss-output-addressed-store-object}\
-    A store object whose store path hashes its content.  This
-    includes derivations, the outputs of
-    [content-addressed derivations](#gloss-content-addressed-derivation),
-    and the outputs of
-    [fixed-output derivations](#gloss-fixed-output-derivation).
+    A [store object] whose [store path] is determined by its contents.
+    This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
   - [substitute]{#gloss-substitute}\
     A substitute is a command invocation stored in the [Nix database] that

From 904878d6d2a4845df472092146d10619b90b7083 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 8 Mar 2022 22:21:41 +0000
Subject: [PATCH 020/183] Move `worker_proto` defs out of `remote-store.cc` to
 own file

These items are not templates, and they declared in
`worker-protocol.hh`; therefore they should live in a
`worker-protocol.cc`.

Anything else needlessly diverges from convention. After all, it is not
like this code is only used in `remote-store.cc`; it is also used in
`daemon.cc`. There is no good reason to place it with the client
implementation or the server implementation when it used equally by
both.
---
 src/libstore/remote-store.cc    | 183 ------------------------------
 src/libstore/worker-protocol.cc | 192 ++++++++++++++++++++++++++++++++
 2 files changed, 192 insertions(+), 183 deletions(-)
 create mode 100644 src/libstore/worker-protocol.cc

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 0ed17a6ce..988b47473 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -18,189 +18,6 @@
 
 namespace nix {
 
-namespace worker_proto {
-
-std::string read(const Store & store, Source & from, Phantom<std::string> _)
-{
-    return readString(from);
-}
-
-void write(const Store & store, Sink & out, const std::string & str)
-{
-    out << str;
-}
-
-
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
-{
-    return store.parseStorePath(readString(from));
-}
-
-void write(const Store & store, Sink & out, const StorePath & storePath)
-{
-    out << store.printStorePath(storePath);
-}
-
-
-std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
-{
-    auto temp = readNum<uint8_t>(from);
-    switch (temp) {
-        case 0:
-            return std::nullopt;
-        case 1:
-            return { Trusted };
-        case 2:
-            return { NotTrusted };
-        default:
-            throw Error("Invalid trusted status from remote");
-    }
-}
-
-void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
-{
-    if (!optTrusted)
-        out << (uint8_t)0;
-    else {
-        switch (*optTrusted) {
-        case Trusted:
-            out << (uint8_t)1;
-            break;
-        case NotTrusted:
-            out << (uint8_t)2;
-            break;
-        default:
-            assert(false);
-        };
-    }
-}
-
-
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
-{
-    return ContentAddress::parse(readString(from));
-}
-
-void write(const Store & store, Sink & out, const ContentAddress & ca)
-{
-    out << renderContentAddress(ca);
-}
-
-
-DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
-{
-    auto s = readString(from);
-    return DerivedPath::parseLegacy(store, s);
-}
-
-void write(const Store & store, Sink & out, const DerivedPath & req)
-{
-    out << req.to_string_legacy(store);
-}
-
-
-Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
-{
-    std::string rawInput = readString(from);
-    return Realisation::fromJSON(
-        nlohmann::json::parse(rawInput),
-        "remote-protocol"
-    );
-}
-
-void write(const Store & store, Sink & out, const Realisation & realisation)
-{
-    out << realisation.toJSON().dump();
-}
-
-
-DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
-{
-    return DrvOutput::parse(readString(from));
-}
-
-void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
-{
-    out << drvOutput.to_string();
-}
-
-
-KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
-{
-    auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
-    auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
-    return KeyedBuildResult {
-        std::move(br),
-        /* .path = */ std::move(path),
-    };
-}
-
-void write(const Store & store, Sink & to, const KeyedBuildResult & res)
-{
-    worker_proto::write(store, to, res.path);
-    worker_proto::write(store, to, static_cast<const BuildResult &>(res));
-}
-
-
-BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
-{
-    BuildResult res;
-    res.status = (BuildResult::Status) readInt(from);
-    from
-        >> res.errorMsg
-        >> res.timesBuilt
-        >> res.isNonDeterministic
-        >> res.startTime
-        >> res.stopTime;
-    auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
-    for (auto && [output, realisation] : builtOutputs)
-        res.builtOutputs.insert_or_assign(
-            std::move(output.outputName),
-            std::move(realisation));
-    return res;
-}
-
-void write(const Store & store, Sink & to, const BuildResult & res)
-{
-    to
-        << res.status
-        << res.errorMsg
-        << res.timesBuilt
-        << res.isNonDeterministic
-        << res.startTime
-        << res.stopTime;
-    DrvOutputs builtOutputs;
-    for (auto & [output, realisation] : res.builtOutputs)
-        builtOutputs.insert_or_assign(realisation.id, realisation);
-    worker_proto::write(store, to, builtOutputs);
-}
-
-
-std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
-{
-    auto s = readString(from);
-    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
-}
-
-void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
-{
-    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
-}
-
-
-std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
-{
-    return ContentAddress::parseOpt(readString(from));
-}
-
-void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
-{
-    out << (caOpt ? renderContentAddress(*caOpt) : "");
-}
-
-}
-
-
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
new file mode 100644
index 000000000..cb0f3f321
--- /dev/null
+++ b/src/libstore/worker-protocol.cc
@@ -0,0 +1,192 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "path-with-outputs.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+#include "worker-protocol.hh"
+#include "archive.hh"
+#include "derivations.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::worker_proto {
+
+std::string read(const Store & store, Source & from, Phantom<std::string> _)
+{
+    return readString(from);
+}
+
+void write(const Store & store, Sink & out, const std::string & str)
+{
+    out << str;
+}
+
+
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
+{
+    return store.parseStorePath(readString(from));
+}
+
+void write(const Store & store, Sink & out, const StorePath & storePath)
+{
+    out << store.printStorePath(storePath);
+}
+
+
+std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
+{
+    auto temp = readNum<uint8_t>(from);
+    switch (temp) {
+        case 0:
+            return std::nullopt;
+        case 1:
+            return { Trusted };
+        case 2:
+            return { NotTrusted };
+        default:
+            throw Error("Invalid trusted status from remote");
+    }
+}
+
+void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
+{
+    if (!optTrusted)
+        out << (uint8_t)0;
+    else {
+        switch (*optTrusted) {
+        case Trusted:
+            out << (uint8_t)1;
+            break;
+        case NotTrusted:
+            out << (uint8_t)2;
+            break;
+        default:
+            assert(false);
+        };
+    }
+}
+
+
+ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
+{
+    return ContentAddress::parse(readString(from));
+}
+
+void write(const Store & store, Sink & out, const ContentAddress & ca)
+{
+    out << renderContentAddress(ca);
+}
+
+
+DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
+{
+    auto s = readString(from);
+    return DerivedPath::parseLegacy(store, s);
+}
+
+void write(const Store & store, Sink & out, const DerivedPath & req)
+{
+    out << req.to_string_legacy(store);
+}
+
+
+Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
+{
+    std::string rawInput = readString(from);
+    return Realisation::fromJSON(
+        nlohmann::json::parse(rawInput),
+        "remote-protocol"
+    );
+}
+
+void write(const Store & store, Sink & out, const Realisation & realisation)
+{
+    out << realisation.toJSON().dump();
+}
+
+
+DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
+{
+    return DrvOutput::parse(readString(from));
+}
+
+void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+{
+    out << drvOutput.to_string();
+}
+
+
+KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
+{
+    auto path = read(store, from, Phantom<DerivedPath> {});
+    auto br = read(store, from, Phantom<BuildResult> {});
+    return KeyedBuildResult {
+        std::move(br),
+        /* .path = */ std::move(path),
+    };
+}
+
+void write(const Store & store, Sink & to, const KeyedBuildResult & res)
+{
+    write(store, to, res.path);
+    write(store, to, static_cast<const BuildResult &>(res));
+}
+
+
+BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
+{
+    BuildResult res;
+    res.status = (BuildResult::Status) readInt(from);
+    from
+        >> res.errorMsg
+        >> res.timesBuilt
+        >> res.isNonDeterministic
+        >> res.startTime
+        >> res.stopTime;
+    auto builtOutputs = read(store, from, Phantom<DrvOutputs> {});
+    for (auto && [output, realisation] : builtOutputs)
+        res.builtOutputs.insert_or_assign(
+            std::move(output.outputName),
+            std::move(realisation));
+    return res;
+}
+
+void write(const Store & store, Sink & to, const BuildResult & res)
+{
+    to
+        << res.status
+        << res.errorMsg
+        << res.timesBuilt
+        << res.isNonDeterministic
+        << res.startTime
+        << res.stopTime;
+    DrvOutputs builtOutputs;
+    for (auto & [output, realisation] : res.builtOutputs)
+        builtOutputs.insert_or_assign(realisation.id, realisation);
+    write(store, to, builtOutputs);
+}
+
+
+std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
+{
+    auto s = readString(from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+{
+    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
+
+std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
+{
+    return ContentAddress::parseOpt(readString(from));
+}
+
+void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+{
+    out << (caOpt ? renderContentAddress(*caOpt) : "");
+}
+
+}

From e7c1113a37e6a8fd0dc2dde0d070dbef276a0481 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 May 2023 17:31:33 -0400
Subject: [PATCH 021/183] Add test for `downstreamPlaceholder`

This is good in general, but in particular ensures when we heavily
refactor it in the next commit there is less likelihood for an
unintentional change in behavior to sneak in.
---
 src/libstore/tests/downstream-placeholder.cc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 src/libstore/tests/downstream-placeholder.cc

diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc
new file mode 100644
index 000000000..7b8e2c649
--- /dev/null
+++ b/src/libstore/tests/downstream-placeholder.cc
@@ -0,0 +1,16 @@
+#include <gtest/gtest.h>
+
+#include "derivations.hh"
+
+namespace nix {
+
+TEST(Derivation, downstreamPlaceholder) {
+    ASSERT_EQ(
+        downstreamPlaceholder(
+            (const Store &)*(const Store *)nullptr, // argument is unused
+            StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
+            "out"),
+        "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
+}
+
+}

From b9e5ce4a27f4a8bbee1a2eeb6fddbf569cbfdd7a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 May 2023 18:01:41 -0400
Subject: [PATCH 022/183] Upgrade `downstreamPlaceholder` to a type with
 methods

This gets us ready for dynamic derivation dependencies (part of RFC 92).
---
 src/libexpr/eval.cc                          |  5 +-
 src/libexpr/eval.hh                          |  4 +-
 src/libexpr/primops.cc                       |  3 +-
 src/libstore/derivations.cc                  | 11 +--
 src/libstore/derivations.hh                  | 12 +---
 src/libstore/downstream-placeholder.cc       | 39 ++++++++++
 src/libstore/downstream-placeholder.hh       | 75 ++++++++++++++++++++
 src/libstore/tests/downstream-placeholder.cc | 27 +++++--
 src/libutil/experimental-features.cc         |  3 +
 src/nix/app.cc                               |  3 +-
 10 files changed, 152 insertions(+), 30 deletions(-)
 create mode 100644 src/libstore/downstream-placeholder.cc
 create mode 100644 src/libstore/downstream-placeholder.hh

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 740a5e677..585670e69 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "filetransfer.hh"
@@ -1058,7 +1059,7 @@ void EvalState::mkOutputString(
             ? store->printStorePath(*std::move(optOutputPath))
             /* Downstream we would substitute this for an actual path once
                we build the floating CA derivation */
-            : downstreamPlaceholder(*store, drvPath, outputName),
+            : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(),
         NixStringContext {
             NixStringContextElem::Built {
                 .drvPath = drvPath,
@@ -2380,7 +2381,7 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str
             // This is testing for the case of CA derivations
             auto sExpected = optOutputPath
                 ? store->printStorePath(*optOutputPath)
-                : downstreamPlaceholder(*store, b.drvPath, output);
+                : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
             if (s != sExpected)
                 error(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index a90ff34c0..62b380929 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -483,7 +483,7 @@ public:
      * Coerce to `DerivedPath`.
      *
      * Must be a string which is either a literal store path or a
-     * "placeholder (see `downstreamPlaceholder()`).
+     * "placeholder (see `DownstreamPlaceholder`).
      *
      * Even more importantly, the string context must be exactly one
      * element, which is either a `NixStringContextElem::Opaque` or
@@ -622,7 +622,7 @@ public:
      * @param optOutputPath Optional output path for that string. Must
      * be passed if and only if output store object is input-addressed.
      * Will be printed to form string if passed, otherwise a placeholder
-     * will be used (see `downstreamPlaceholder()`).
+     * will be used (see `DownstreamPlaceholder`).
      */
     void mkOutputString(
         Value & value,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6fbd66389..cfae1e5f8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1,5 +1,6 @@
 #include "archive.hh"
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "globals.hh"
@@ -87,7 +88,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         auto outputs = resolveDerivedPath(*store, drv);
         for (auto & [outputName, outputPath] : outputs) {
             res.insert_or_assign(
-                downstreamPlaceholder(*store, drv.drvPath, outputName),
+                DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(),
                 store->printStorePath(outputPath)
             );
         }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index d56dc727b..56a3df66d 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -1,4 +1,5 @@
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
@@ -810,13 +811,7 @@ std::string hashPlaceholder(const std::string_view outputName)
     return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
 }
 
-std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
-{
-    auto drvNameWithExtension = drvPath.name();
-    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
-    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
-    return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
-}
+
 
 
 static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
@@ -880,7 +875,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
         for (auto & outputName : inputOutputs) {
             if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
                 inputRewrites.emplace(
-                    downstreamPlaceholder(store, inputDrv, outputName),
+                    DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
                     store.printStorePath(*actualPath));
                 resolved.inputSrcs.insert(*actualPath);
             } else {
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 1e2143f31..fa79f77fd 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -6,6 +6,7 @@
 #include "hash.hh"
 #include "content-address.hh"
 #include "repair-flag.hh"
+#include "derived-path.hh"
 #include "sync.hh"
 #include "comparator.hh"
 
@@ -495,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
  */
 std::string hashPlaceholder(const std::string_view outputName);
 
-/**
- * This creates an opaque and almost certainly unique string
- * deterministically from a derivation path and output name.
- *
- * It is used as a placeholder to allow derivations to refer to
- * content-addressed paths whose content --- and thus the path
- * themselves --- isn't yet known. This occurs when a derivation has a
- * dependency which is a CA derivation.
- */
-std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
-
 extern const Hash impureOutputHash;
 
 }
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
new file mode 100644
index 000000000..1752738f2
--- /dev/null
+++ b/src/libstore/downstream-placeholder.cc
@@ -0,0 +1,39 @@
+#include "downstream-placeholder.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+std::string DownstreamPlaceholder::render() const
+{
+    return "/" + hash.to_string(Base32, false);
+}
+
+
+DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
+    const StorePath & drvPath,
+    std::string_view outputName)
+{
+    auto drvNameWithExtension = drvPath.name();
+    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
+    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
+    return DownstreamPlaceholder {
+        hashString(htSHA256, clearText)
+    };
+}
+
+DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
+    const DownstreamPlaceholder & placeholder,
+    std::string_view outputName,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::DynamicDerivations);
+    auto compressed = compressHash(placeholder.hash, 20);
+    auto clearText = "nix-computed-output:"
+        + compressed.to_string(Base32, false)
+        + ":" + std::string { outputName };
+    return DownstreamPlaceholder {
+        hashString(htSHA256, clearText)
+    };
+}
+
+}
diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh
new file mode 100644
index 000000000..f0c0dee77
--- /dev/null
+++ b/src/libstore/downstream-placeholder.hh
@@ -0,0 +1,75 @@
+#pragma once
+///@file
+
+#include "hash.hh"
+#include "path.hh"
+
+namespace nix {
+
+/**
+ * Downstream Placeholders are opaque and almost certainly unique values
+ * used to allow derivations to refer to store objects which are yet to
+ * be built and for we do not yet have store paths for.
+ *
+ * They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`,
+ * except for the cases involving input addressing or fixed outputs
+ * where we do know a store path for the derivation output in advance.
+ *
+ * Unlike `DerivationPath`, however, `DownstreamPlaceholder` is
+ * purposefully opaque and obfuscated. This is so they are hard to
+ * create by accident, and so substituting them (once we know what the
+ * path to store object is) is unlikely to capture other stuff it
+ * shouldn't.
+ *
+ * We use them with `Derivation`: the `render()` method is called to
+ * render an opaque string which can be used in the derivation, and the
+ * resolving logic can substitute those strings for store paths when
+ * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`.
+ */
+class DownstreamPlaceholder
+{
+    /**
+     * `DownstreamPlaceholder` is just a newtype of `Hash`.
+     * This its only field.
+     */
+    Hash hash;
+
+    /**
+     * Newtype constructor
+     */
+    DownstreamPlaceholder(Hash hash) : hash(hash) { }
+
+public:
+    /**
+     * This creates an opaque and almost certainly unique string
+     * deterministically from the placeholder.
+     */
+    std::string render() const;
+
+    /**
+     * Create a placeholder for an unknown output of a content-addressed
+     * derivation.
+     *
+     * The derivation itself is known (we have a store path for it), but
+     * the output doesn't yet have a known store path.
+     */
+    static DownstreamPlaceholder unknownCaOutput(
+        const StorePath & drvPath,
+        std::string_view outputName);
+
+    /**
+     * Create a placehold for the output of an unknown derivation.
+     *
+     * The derivation is not yet known because it is a dynamic
+     * derivaiton --- it is itself an output of another derivation ---
+     * and we just have (another) placeholder for it.
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
+    static DownstreamPlaceholder unknownDerivation(
+        const DownstreamPlaceholder & drvPlaceholder,
+        std::string_view outputName,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+};
+
+}
diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc
index 7b8e2c649..ec3e1000f 100644
--- a/src/libstore/tests/downstream-placeholder.cc
+++ b/src/libstore/tests/downstream-placeholder.cc
@@ -1,16 +1,33 @@
 #include <gtest/gtest.h>
 
-#include "derivations.hh"
+#include "downstream-placeholder.hh"
 
 namespace nix {
 
-TEST(Derivation, downstreamPlaceholder) {
+TEST(DownstreamPlaceholder, unknownCaOutput) {
     ASSERT_EQ(
-        downstreamPlaceholder(
-            (const Store &)*(const Store *)nullptr, // argument is unused
+        DownstreamPlaceholder::unknownCaOutput(
             StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
-            "out"),
+            "out").render(),
         "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
 }
 
+TEST(DownstreamPlaceholder, unknownDerivation) {
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+    mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+
+    ASSERT_EQ(
+        DownstreamPlaceholder::unknownDerivation(
+            DownstreamPlaceholder::unknownCaOutput(
+                StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
+                "out"),
+            "out",
+            mockXpSettings).render(),
+        "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");
+}
+
 }
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index ad0ec0427..5aae0347b 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -207,6 +207,9 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
 
               - "text hashing" derivation outputs, so we can build .drv
                 files.
+
+              - dependencies in derivations on the outputs of
+                derivations that are themselves derivations outputs.
         )",
     },
 }};
diff --git a/src/nix/app.cc b/src/nix/app.cc
index fd4569bb4..e678b54f0 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -7,6 +7,7 @@
 #include "names.hh"
 #include "command.hh"
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 
 namespace nix {
 
@@ -23,7 +24,7 @@ StringPairs resolveRewrites(
         if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
             for (auto & [ outputName, outputPath ] : drvDep->outputs)
                 res.emplace(
-                    downstreamPlaceholder(store, drvDep->drvPath, outputName),
+                    DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(),
                     store.printStorePath(outputPath)
                 );
     return res;

From cb5052d98fa9a5d64d1700fe434c1c37a72e45d1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 May 2023 22:04:59 -0400
Subject: [PATCH 023/183] Revert "Revert "Use template structs instead of
 phantoms""

This is the more typically way to do [Argument-dependent
lookup](https://en.cppreference.com/w/cpp/language/adl)-leveraging
generic serializers in C++. It makes the relationship between the `read`
and `write` methods more clear and rigorous, and also looks more
familiar to users coming from other languages that do not have C++'s
libertine ad-hoc overloading.

I am returning to this because during the review in
https://github.com/NixOS/nix/pull/6223, it came up as something that
would make the code easier to read --- easier today hopefully already,
but definitely easier if we were have multiple codified protocols with
code sharing between them as that PR seeks to accomplish.

If I recall correctly, the main criticism of this the first time around
(in 2020) was that having to specify the type when writing, e.g.
`WorkerProto<MyType>::write`, was too verbose and cumbersome. This is
now addressed with the `workerProtoWrite` wrapper function.

This method is also the way `nlohmann::json`, which we have used for a
number of years now, does its serializers, for what its worth.

This reverts commit 45a0ed82f089158a79c8c25ef844c55e4a74fc35. That
commit in turn reverted 9ab07e99f527d1fa3adfa02839da477a1528d64b.
---
 src/libstore/build/derivation-goal.cc |   4 +-
 src/libstore/daemon.cc                |  50 ++++++------
 src/libstore/derivations.cc           |   4 +-
 src/libstore/export-import.cc         |   4 +-
 src/libstore/legacy-ssh-store.cc      |  16 ++--
 src/libstore/path-info.cc             |   4 +-
 src/libstore/remote-store.cc          |  62 +++++++-------
 src/libstore/worker-protocol.cc       |  58 ++++++-------
 src/libstore/worker-protocol.hh       | 113 ++++++++++++++++----------
 src/nix-store/nix-store.cc            |  18 ++--
 10 files changed, 182 insertions(+), 151 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 5b1c923cd..df7d21e54 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1152,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
+    workerProtoWrite(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1163,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        worker_proto::write(worker.store, hook->sink, missingOutputs);
+        workerProtoWrite(worker.store, hook->sink, missingOutputs);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 5083497a9..b6dd83684 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -263,7 +263,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
 {
     std::vector<DerivedPath> reqs;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
-        reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
+        reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
     } else {
         for (auto & s : readStrings<Strings>(from))
             reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
@@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
 
         SubstituteFlag substitute = NoSubstitute;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
-        worker_proto::write(*store, to, res);
+        workerProtoWrite(*store, to, res);
         break;
     }
 
@@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        worker_proto::write(*store, to, res);
+        workerProtoWrite(*store, to, res);
         break;
     }
 
@@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        worker_proto::write(*store, to, paths);
+        workerProtoWrite(*store, to, paths);
         break;
     }
 
@@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        worker_proto::write(*store, to, outputs);
+        workerProtoWrite(*store, to, outputs);
         break;
     }
 
@@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto refs = WorkerProto<StorePathSet>::read(*store, from);
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         std::string suffix = readString(from);
         std::string s = readString(from);
-        auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto refs = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -567,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto results = store->buildPathsWithResults(drvs, mode);
         logger->stopWork();
 
-        worker_proto::write(*store, to, results);
+        workerProtoWrite(*store, to, results);
 
         break;
     }
@@ -644,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             DrvOutputs builtOutputs;
             for (auto & [output, realisation] : res.builtOutputs)
                 builtOutputs.insert_or_assign(realisation.id, realisation);
-            worker_proto::write(*store, to, builtOutputs);
+            workerProtoWrite(*store, to, builtOutputs);
         }
         break;
     }
@@ -709,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -779,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            worker_proto::write(*store, to, i->second.references);
+            workerProtoWrite(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -790,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto paths = WorkerProto<StorePathSet>::read(*store, from);
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
+            pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -802,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            worker_proto::write(*store, to, i.second.references);
+            workerProtoWrite(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -812,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        worker_proto::write(*store, to, paths);
+        workerProtoWrite(*store, to, paths);
         break;
     }
 
@@ -884,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        info.references = WorkerProto<StorePathSet>::read(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = ContentAddress::parseOpt(readString(from));
@@ -935,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        worker_proto::write(*store, to, willBuild);
-        worker_proto::write(*store, to, willSubstitute);
-        worker_proto::write(*store, to, unknown);
+        workerProtoWrite(*store, to, willBuild);
+        workerProtoWrite(*store, to, willSubstitute);
+        workerProtoWrite(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
@@ -950,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             store->registerDrvOutput(Realisation{
                 .id = outputId, .outPath = outputPath});
         } else {
-            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            auto realisation = WorkerProto<Realisation>::read(*store, from);
             store->registerDrvOutput(realisation);
         }
         logger->stopWork();
@@ -965,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             std::set<StorePath> outPaths;
             if (info) outPaths.insert(info->outPath);
-            worker_proto::write(*store, to, outPaths);
+            workerProtoWrite(*store, to, outPaths);
         } else {
             std::set<Realisation> realisations;
             if (info) realisations.insert(*info);
-            worker_proto::write(*store, to, realisations);
+            workerProtoWrite(*store, to, realisations);
         }
         break;
     }
@@ -1045,7 +1045,7 @@ void processConnection(
         auto temp = trusted
             ? store->isTrustedClient()
             : std::optional { NotTrusted };
-        worker_proto::write(*store, to, temp);
+        workerProtoWrite(*store, to, temp);
     }
 
     /* Send startup error messages to the client. */
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index d56dc727b..15d3e07b3 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -748,7 +748,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
+    drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -796,7 +796,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.raw());
     }
-    worker_proto::write(store, out, drv.inputSrcs);
+    workerProtoWrite(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 4eb838b68..5ea263a86 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    worker_proto::write(*this, teeSink, info->references);
+    workerProtoWrite(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
 
-        auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
+        auto references = WorkerProto<StorePathSet>::read(*this, source);
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 2012584e0..9a470a99e 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            worker_proto::write(*this, conn->to, info.references);
+            workerProtoWrite(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, conn->to, info.references);
+            workerProtoWrite(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -294,7 +294,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+            auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -358,10 +358,10 @@ public:
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+        for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
             out.insert(i);
     }
 
@@ -374,10 +374,10 @@ public:
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn->to.flush();
 
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 
     void connect() override
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index e60d7abe0..9dd4a3ae7 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -131,7 +131,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
     auto narHash = Hash::parseAny(readString(source), htSHA256);
     ValidPathInfo info(path, narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
+    info.references = WorkerProto<StorePathSet>::read(store, source);
     source >> info.registrationTime >> info.narSize;
     if (format >= 16) {
         source >> info.ultimate;
@@ -152,7 +152,7 @@ void ValidPathInfo::write(
         sink << store.printStorePath(path);
     sink << (deriver ? store.printStorePath(*deriver) : "")
          << narHash.to_string(Base16, false);
-    worker_proto::write(store, sink, references);
+    workerProtoWrite(store, sink, references);
     sink << registrationTime << narSize;
     if (format >= 16) {
         sink << ultimate
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 988b47473..c3dfb5979 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -100,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn)
         }
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
-            conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
+            conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
         } else {
             // We don't know the answer; protocol to old.
             conn.remoteTrustsUs = std::nullopt;
@@ -227,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
             conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
         }
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -242,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -259,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -283,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -296,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            worker_proto::write(*this, conn->to, paths);
+            workerProtoWrite(*this, conn->to, paths);
         } else
-            worker_proto::write(*this, conn->to, pathsMap);
+            workerProtoWrite(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -306,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -349,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+    for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
         referrers.insert(i);
 }
 
@@ -359,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -371,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -381,7 +381,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
         auto conn(getConnection());
         conn->to << wopQueryDerivationOutputMap << printStorePath(path);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+        return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
     } else {
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -427,7 +427,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             << wopAddToStore
             << name
             << caMethod.render(hashType);
-        worker_proto::write(*this, conn->to, references);
+        workerProtoWrite(*this, conn->to, references);
         conn->to << repair;
 
         // The dump source may invoke the store, so we need to make some room.
@@ -452,7 +452,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                         name, printHashType(hashType));
                 std::string s = dump.drain();
                 conn->to << wopAddTextToStore << name << s;
-                worker_proto::write(*this, conn->to, references);
+                workerProtoWrite(*this, conn->to, references);
                 conn.processStderr();
             },
             [&](const FileIngestionMethod & fim) -> void {
@@ -518,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, sink, info.references);
+            workerProtoWrite(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -528,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
         assert(importedPaths.size() <= 1);
     }
 
@@ -537,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        worker_proto::write(*this, conn->to, info.references);
+        workerProtoWrite(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -610,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
         conn->to << info.id.to_string();
         conn->to << std::string(info.outPath.to_string());
     } else {
-        worker_proto::write(*this, conn->to, info);
+        workerProtoWrite(*this, conn->to, info);
     }
     conn.processStderr();
 }
@@ -632,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
 
         auto real = [&]() -> std::shared_ptr<const Realisation> {
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-                auto outPaths = worker_proto::read(
-                    *this, conn->from, Phantom<std::set<StorePath>> {});
+                auto outPaths = WorkerProto<std::set<StorePath>>::read(
+                    *this, conn->from);
                 if (outPaths.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
             } else {
-                auto realisations = worker_proto::read(
-                    *this, conn->from, Phantom<std::set<Realisation>> {});
+                auto realisations = WorkerProto<std::set<Realisation>>::read(
+                    *this, conn->from);
                 if (realisations.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(*realisations.begin());
@@ -653,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
 {
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
-        worker_proto::write(store, conn->to, reqs);
+        workerProtoWrite(store, conn->to, reqs);
     } else {
         Strings ss;
         for (auto & p : reqs) {
@@ -723,7 +723,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
         writeDerivedPaths(*this, conn, paths);
         conn->to << buildMode;
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
+        return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
     } else {
         // Avoid deadlock.
         conn_.reset();
@@ -806,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
         conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
-        auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
         for (auto && [output, realisation] : builtOutputs)
             res.builtOutputs.insert_or_assign(
                 std::move(output.outputName),
@@ -865,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    worker_proto::write(*this, conn->to, options.pathsToDelete);
+    workerProtoWrite(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -924,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
         conn->to << wopQueryMissing;
         writeDerivedPaths(*this, conn, targets);
         conn.processStderr();
-        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
+        willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
+        unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index cb0f3f321..51bb12026 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -9,31 +9,31 @@
 
 #include <nlohmann/json.hpp>
 
-namespace nix::worker_proto {
+namespace nix {
 
-std::string read(const Store & store, Source & from, Phantom<std::string> _)
+std::string WorkerProto<std::string>::read(const Store & store, Source & from)
 {
     return readString(from);
 }
 
-void write(const Store & store, Sink & out, const std::string & str)
+void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
 {
     out << str;
 }
 
 
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
+StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
 {
     return store.parseStorePath(readString(from));
 }
 
-void write(const Store & store, Sink & out, const StorePath & storePath)
+void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
 {
     out << store.printStorePath(storePath);
 }
 
 
-std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
+std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
 {
     auto temp = readNum<uint8_t>(from);
     switch (temp) {
@@ -48,7 +48,7 @@ std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std:
     }
 }
 
-void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
         out << (uint8_t)0;
@@ -67,30 +67,30 @@ void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & o
 }
 
 
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
+ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
 {
     return ContentAddress::parse(readString(from));
 }
 
-void write(const Store & store, Sink & out, const ContentAddress & ca)
+void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
 {
     out << renderContentAddress(ca);
 }
 
 
-DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
+DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
 {
     auto s = readString(from);
     return DerivedPath::parseLegacy(store, s);
 }
 
-void write(const Store & store, Sink & out, const DerivedPath & req)
+void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
 {
     out << req.to_string_legacy(store);
 }
 
 
-Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
+Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
 {
     std::string rawInput = readString(from);
     return Realisation::fromJSON(
@@ -99,41 +99,41 @@ Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
     );
 }
 
-void write(const Store & store, Sink & out, const Realisation & realisation)
+void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
 {
     out << realisation.toJSON().dump();
 }
 
 
-DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
+DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
 {
     return DrvOutput::parse(readString(from));
 }
 
-void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
 {
     out << drvOutput.to_string();
 }
 
 
-KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
+KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
 {
-    auto path = read(store, from, Phantom<DerivedPath> {});
-    auto br = read(store, from, Phantom<BuildResult> {});
+    auto path = WorkerProto<DerivedPath>::read(store, from);
+    auto br = WorkerProto<BuildResult>::read(store, from);
     return KeyedBuildResult {
         std::move(br),
         /* .path = */ std::move(path),
     };
 }
 
-void write(const Store & store, Sink & to, const KeyedBuildResult & res)
+void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
 {
-    write(store, to, res.path);
-    write(store, to, static_cast<const BuildResult &>(res));
+    workerProtoWrite(store, to, res.path);
+    workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
+BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
 {
     BuildResult res;
     res.status = (BuildResult::Status) readInt(from);
@@ -143,7 +143,7 @@ BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
         >> res.isNonDeterministic
         >> res.startTime
         >> res.stopTime;
-    auto builtOutputs = read(store, from, Phantom<DrvOutputs> {});
+    auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
     for (auto && [output, realisation] : builtOutputs)
         res.builtOutputs.insert_or_assign(
             std::move(output.outputName),
@@ -151,7 +151,7 @@ BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
     return res;
 }
 
-void write(const Store & store, Sink & to, const BuildResult & res)
+void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
 {
     to
         << res.status
@@ -163,28 +163,28 @@ void write(const Store & store, Sink & to, const BuildResult & res)
     DrvOutputs builtOutputs;
     for (auto & [output, realisation] : res.builtOutputs)
         builtOutputs.insert_or_assign(realisation.id, realisation);
-    write(store, to, builtOutputs);
+    workerProtoWrite(store, to, builtOutputs);
 }
 
 
-std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
+std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
 {
     auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
 {
     out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
+std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
 {
     return ContentAddress::parseOpt(readString(from));
 }
 
-void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
 {
     out << (caOpt ? renderContentAddress(*caOpt) : "");
 }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 34b2fc17b..28fd1a462 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -80,40 +80,71 @@ class Store;
 struct Source;
 
 /**
- * Used to guide overloading
+ * Data type for canonical pairs of serializers for the worker protocol.
  *
  * See https://en.cppreference.com/w/cpp/language/adl for the broader
  * concept of what is going on here.
  */
 template<typename T>
-struct Phantom {};
+struct WorkerProto {
+    static T read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const T & t);
+};
 
+/**
+ * Wrapper function around `WorkerProto<T>::write` that allows us to
+ * infer the type instead of having to write it down explicitly.
+ */
+template<typename T>
+void workerProtoWrite(const Store & store, Sink & out, const T & t)
+{
+    WorkerProto<T>::write(store, out, t);
+}
 
-namespace worker_proto {
-/* FIXME maybe move more stuff inside here */
+/**
+ * Declare a canonical serializer pair for the worker protocol.
+ *
+ * We specialize the struct merely to indicate that we are implementing
+ * the function for the given type.
+ *
+ * Some sort of `template<...>` must be used with the caller for this to
+ * be legal specialization syntax. See below for what that looks like in
+ * practice.
+ */
+#define MAKE_WORKER_PROTO(T) \
+    struct WorkerProto< T > { \
+        static T read(const Store & store, Source & from); \
+        static void write(const Store & store, Sink & out, const T & t); \
+    };
 
-#define MAKE_WORKER_PROTO(TEMPLATE, T) \
-    TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
-    TEMPLATE void write(const Store & store, Sink & out, const T & str)
+template<>
+MAKE_WORKER_PROTO(std::string);
+template<>
+MAKE_WORKER_PROTO(StorePath);
+template<>
+MAKE_WORKER_PROTO(ContentAddress);
+template<>
+MAKE_WORKER_PROTO(DerivedPath);
+template<>
+MAKE_WORKER_PROTO(Realisation);
+template<>
+MAKE_WORKER_PROTO(DrvOutput);
+template<>
+MAKE_WORKER_PROTO(BuildResult);
+template<>
+MAKE_WORKER_PROTO(KeyedBuildResult);
+template<>
+MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
 
-MAKE_WORKER_PROTO(, std::string);
-MAKE_WORKER_PROTO(, StorePath);
-MAKE_WORKER_PROTO(, ContentAddress);
-MAKE_WORKER_PROTO(, DerivedPath);
-MAKE_WORKER_PROTO(, Realisation);
-MAKE_WORKER_PROTO(, DrvOutput);
-MAKE_WORKER_PROTO(, BuildResult);
-MAKE_WORKER_PROTO(, KeyedBuildResult);
-MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
+template<typename T>
+MAKE_WORKER_PROTO(std::vector<T>);
+template<typename T>
+MAKE_WORKER_PROTO(std::set<T>);
 
-MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
-MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
-
-#define X_ template<typename K, typename V>
-#define Y_ std::map<K, V>
-MAKE_WORKER_PROTO(X_, Y_);
+template<typename K, typename V>
+#define X_ std::map<K, V>
+MAKE_WORKER_PROTO(X_);
 #undef X_
-#undef Y_
 
 /**
  * These use the empty string for the null case, relying on the fact
@@ -129,72 +160,72 @@ MAKE_WORKER_PROTO(X_, Y_);
  * worker protocol harder to implement in other languages where such
  * specializations may not be allowed.
  */
-MAKE_WORKER_PROTO(, std::optional<StorePath>);
-MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
+template<>
+MAKE_WORKER_PROTO(std::optional<StorePath>);
+template<>
+MAKE_WORKER_PROTO(std::optional<ContentAddress>);
 
 template<typename T>
-std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
+std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.push_back(read(store, from, Phantom<T> {}));
+        resSet.push_back(WorkerProto<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void write(const Store & store, Sink & out, const std::vector<T> & resSet)
+void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        write(store, out, key);
+        WorkerProto<T>::write(store, out, key);
     }
 }
 
 template<typename T>
-std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
+std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.insert(read(store, from, Phantom<T> {}));
+        resSet.insert(WorkerProto<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void write(const Store & store, Sink & out, const std::set<T> & resSet)
+void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        write(store, out, key);
+        WorkerProto<T>::write(store, out, key);
     }
 }
 
 template<typename K, typename V>
-std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
+std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(from);
     while (size--) {
-        auto k = read(store, from, Phantom<K> {});
-        auto v = read(store, from, Phantom<V> {});
+        auto k = WorkerProto<K>::read(store, from);
+        auto v = WorkerProto<V>::read(store, from);
         resMap.insert_or_assign(std::move(k), std::move(v));
     }
     return resMap;
 }
 
 template<typename K, typename V>
-void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
 {
     out << resMap.size();
     for (auto & i : resMap) {
-        write(store, out, i.first);
-        write(store, out, i.second);
+        WorkerProto<K>::write(store, out, i.first);
+        WorkerProto<V>::write(store, out, i.second);
     }
 }
 
 }
-
-}
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 40f30eb63..61c189efb 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -849,7 +849,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = WorkerProto<StorePathSet>::read(*store, in);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -858,19 +858,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                     store->substitutePaths(paths);
                 }
 
-                worker_proto::write(*store, out, store->queryValidPaths(paths));
+                workerProtoWrite(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                auto paths = WorkerProto<StorePathSet>::read(*store, in);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        worker_proto::write(*store, out, info->references);
+                        workerProtoWrite(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
+                store->exportPaths(WorkerProto<StorePathSet>::read(*store, in), out);
                 break;
             }
 
@@ -944,7 +944,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                     DrvOutputs builtOutputs;
                     for (auto & [output, realisation] : status.builtOutputs)
                         builtOutputs.insert_or_assign(realisation.id, realisation);
-                    worker_proto::write(*store, out, builtOutputs);
+                    workerProtoWrite(*store, out, builtOutputs);
                 }
 
                 break;
@@ -953,9 +953,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(worker_proto::read(*store, in, Phantom<StorePathSet> {}),
+                store->computeFSClosure(WorkerProto<StorePathSet>::read(*store, in),
                     closure, false, includeOutputs);
-                worker_proto::write(*store, out, closure);
+                workerProtoWrite(*store, out, closure);
                 break;
             }
 
@@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
+                info.references = WorkerProto<StorePathSet>::read(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));

From 9923403d90e4aa4b7a7a7c27840295ba9b659a97 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 8 Mar 2022 22:03:03 +0000
Subject: [PATCH 024/183] Don't use `store-api.hh` in `worker-protocol.hh`

Using abstract types like can help cut down on compilation time, both
from scratch, and especially incremental builds during development. The
idea is that `worker-protocol.hh` can declare all the (de)serializers, but
only again abstract types; when code needs to use some (de)serializers, it can
include headers just for the data types it needs to (de)serialize.

`store-api.hh` in particular is a bit of a sledgehammer, and the data
types we want to serialize have their own headers.
---
 src/libstore/path-info.cc       |  1 +
 src/libstore/worker-protocol.hh | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index e60d7abe0..7eeda35d1 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -1,5 +1,6 @@
 #include "path-info.hh"
 #include "worker-protocol.hh"
+#include "store-api.hh"
 
 namespace nix {
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 34b2fc17b..fa7202ae6 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "store-api.hh"
 #include "serialise.hh"
 
 namespace nix {
@@ -79,6 +78,15 @@ typedef enum {
 class Store;
 struct Source;
 
+// items being serialized
+struct DerivedPath;
+struct DrvOutput;
+struct Realisation;
+struct BuildResult;
+struct KeyedBuildResult;
+enum TrustedFlag : bool;
+
+
 /**
  * Used to guide overloading
  *

From 4c4ae887b871ace7bc8528bc3dcf4fe85a92ff09 Mon Sep 17 00:00:00 2001
From: Konstantin Vukolov <kasyanvukolov@yandex.ru>
Date: Thu, 18 May 2023 13:18:34 +0300
Subject: [PATCH 025/183] Add option isInteractive

---
 src/libfetchers/git.cc | 21 +++++++--------------
 src/libutil/util.cc    | 14 ++++++++++++--
 src/libutil/util.hh    |  3 ++-
 3 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7ec6efa4d..7df467feb 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -22,14 +22,6 @@ namespace nix::fetchers {
 
 namespace {
 
-template<typename... Args>
-auto runProgramWithCredentialsInput(Args... args)
-{
-    logger->pause();
-    Finally defer([]{ logger->resume(); });
-    return runProgram(std::forward<Args>(args)...);
-}
-
 // Explicit initial branch of our bare repo to suppress warnings from new version of git.
 // The value itself does not matter, since we always fetch a specific revision or branch.
 // It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
@@ -67,10 +59,11 @@ Path getCachePath(std::string_view key)
 //   ...
 std::optional<std::string> readHead(const Path & path)
 {
-    auto [status, output] = runProgramWithCredentialsInput(RunOptions {
+    auto [status, output] = runProgram(RunOptions {
         .program = "git",
         // FIXME: use 'HEAD' to avoid returning all refs
         .args = {"ls-remote", "--symref", path},
+        .isInteractive = true,
     });
     if (status != 0) return std::nullopt;
 
@@ -359,7 +352,7 @@ struct GitInputScheme : InputScheme
 
         args.push_back(destDir);
 
-        runProgramWithCredentialsInput("git", true, args);
+        runProgram("git", true, args, {}, true);
     }
 
     std::optional<Path> getSourcePath(const Input & input) override
@@ -564,7 +557,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgramWithCredentialsInput("git", true, Strings { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                    runProgram("git", true, Strings { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -630,8 +623,8 @@ struct GitInputScheme : InputScheme
                 // exists, see FIXME above) so use a big hammer and fetch
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgramWithCredentialsInput("git", true, Strings { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+                runProgram("git", true, Strings { "-C", tmpDir, "fetch", "--quiet", "--force",
+                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
             runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
@@ -658,7 +651,7 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgramWithCredentialsInput("git", true, Strings{ "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+                runProgram("git", true, Strings{ "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 21d1c8dcd..3a8309149 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1141,9 +1141,9 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 }
 
 std::string runProgram(Path program, bool searchPath, const Strings & args,
-    const std::optional<std::string> & input)
+    const std::optional<std::string> & input, bool isInteractive)
 {
-    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
+    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
 
     if (!statusOk(res.first))
         throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
@@ -1193,6 +1193,16 @@ void runProgram2(const RunOptions & options)
     // case), so we can't use it if we alter the environment
     processOptions.allowVfork = !options.environment;
 
+    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
+    if (options.isInteractive) {
+        logger->pause();
+        resumeLoggerDefer.emplace(
+            []() {
+                logger->resume();
+            }
+        );
+    }
+
     /* Fork. */
     Pid pid = startProcess([&]() {
         if (options.environment)
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 040fed68f..a7907cd14 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -415,7 +415,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
  */
 std::string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
-    const std::optional<std::string> & input = {});
+    const std::optional<std::string> & input = {}, bool isInteractive = false);
 
 struct RunOptions
 {
@@ -430,6 +430,7 @@ struct RunOptions
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
     bool mergeStderrToStdout = false;
+    bool isInteractive = false;
 };
 
 std::pair<int, std::string> runProgram(RunOptions && options);

From 31ffd0c1fe1d5112746f3c8d608cbfb4d6290d1b Mon Sep 17 00:00:00 2001
From: Konstantin Vukolov <kasyanvukolov@yandex.ru>
Date: Thu, 18 May 2023 13:26:23 +0300
Subject: [PATCH 026/183] Remove unused

---
 src/libfetchers/git.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7df467feb..47282f6c4 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -1,5 +1,4 @@
 #include "fetchers.hh"
-#include "finally.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
@@ -557,7 +556,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, Strings { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -623,7 +622,7 @@ struct GitInputScheme : InputScheme
                 // exists, see FIXME above) so use a big hammer and fetch
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgram("git", true, Strings { "-C", tmpDir, "fetch", "--quiet", "--force",
+                runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
@@ -651,7 +650,7 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgram("git", true, Strings{ "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
+                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;

From 36b7e30c1184035e72a7f24dc4656099ac6e208e Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Fri, 19 May 2023 22:47:40 +0200
Subject: [PATCH 027/183] Make mounting ssl cert file optional

---
 src/libstore/build/local-derivation-goal.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 36c89bee9..b50ff1f86 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1777,7 +1777,7 @@ void LocalDerivationGoal::runChild()
                     if (pathExists(path))
                         ss.push_back(path);
 
-                dirsInChroot.emplace(settings.caFile, "/etc/ssl/certs/ca-certificates.crt");
+                dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
             }
 
             for (auto & i : ss) dirsInChroot.emplace(i, i);

From b14fea6fffaafa30faf67d89b6b6a01c14b3ddf3 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Fri, 19 May 2023 23:30:35 +0200
Subject: [PATCH 028/183] Shortcircuit for empty caFile

---
 src/libstore/build/local-derivation-goal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index b50ff1f86..05d6685da 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1777,7 +1777,8 @@ void LocalDerivationGoal::runChild()
                     if (pathExists(path))
                         ss.push_back(path);
 
-                dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
+                if (settings.caFile != "")
+                    dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
             }
 
             for (auto & i : ss) dirsInChroot.emplace(i, i);

From 5a98dd0b3911be970dbaae414b150c35241d2523 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Mon, 22 May 2023 02:32:09 +0200
Subject: [PATCH 029/183] Add tests for bind mount of SSL certs in sandbox

---
 tests/linux-sandbox-cert-test.nix | 29 +++++++++++++++++++++++++++++
 tests/linux-sandbox.sh            | 24 ++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 tests/linux-sandbox-cert-test.nix

diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix
new file mode 100644
index 000000000..2b86dad2e
--- /dev/null
+++ b/tests/linux-sandbox-cert-test.nix
@@ -0,0 +1,29 @@
+{ fixed-output }:
+
+with import ./config.nix;
+
+mkDerivation ({
+  name = "ssl-export";
+  buildCommand = ''
+    # Add some indirection, otherwise grepping into the debug output finds the string.
+    report () { echo CERT_$1_IN_SANDBOX; }
+
+    if [ -f /etc/ssl/certs/ca-certificates.crt ]; then
+      content=$(</etc/ssl/certs/ca-certificates.crt)
+      if [ "$content" == CERT_CONTENT ]; then
+        report present
+      fi
+    else
+      report missing
+    fi
+
+    # Always fail, because we do not want to bother with fixed-output
+    # derivations being cached, and do not want to compute the right hash.
+    false;
+  '';
+} // (
+  if fixed-output == "fixed-output"
+  then { outputHash = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; }
+  else { }
+))
+
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 5a2cf7abd..45f0ce7a4 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -40,3 +40,27 @@ grepQuiet 'may not be deterministic' $TEST_ROOT/log
 
 # Test that sandboxed builds cannot write to /etc easily
 (! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store)
+
+
+## Test mounting of SSL certificates into the sandbox
+testCert () {
+    (! nix-build linux-sandbox-cert-test.nix --argstr fixed-output "$2" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$3" 2> $TEST_ROOT/log)
+    cat $TEST_ROOT/log
+    grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log
+}
+
+nocert=$TEST_ROOT/no-cert-file.pem
+cert=$TEST_ROOT/some-cert-file.pem
+echo -n "CERT_CONTENT" > $cert
+
+# No cert in sandbox when not a fixed-output derivation
+testCert missing normal       "$cert"
+
+# No cert in sandbox when ssl-cert-file is empty
+testCert missing fixed-output ""
+
+# No cert in sandbox when ssl-cert-file is a nonexistent file
+testCert missing fixed-output "$nocert"
+
+# Cert in sandbox when ssl-cert-file is set to an existing file
+testCert present fixed-output "$cert"

From b9c2f834ee37c76fcb21bc5dbcd60bb58a229194 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Mon, 22 May 2023 23:39:31 +0200
Subject: [PATCH 030/183] Check exact error codes in linux-sandbox.sh

---
 tests/linux-sandbox-cert-test.nix | 45 ++++++++++++++++---------------
 tests/linux-sandbox.sh            | 18 ++++++++-----
 2 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix
index 2b86dad2e..2fc083ea9 100644
--- a/tests/linux-sandbox-cert-test.nix
+++ b/tests/linux-sandbox-cert-test.nix
@@ -1,29 +1,30 @@
-{ fixed-output }:
+{ mode }:
 
 with import ./config.nix;
 
-mkDerivation ({
-  name = "ssl-export";
-  buildCommand = ''
-    # Add some indirection, otherwise grepping into the debug output finds the string.
-    report () { echo CERT_$1_IN_SANDBOX; }
+mkDerivation (
+  {
+    name = "ssl-export";
+    buildCommand = ''
+      # Add some indirection, otherwise grepping into the debug output finds the string.
+      report () { echo CERT_$1_IN_SANDBOX; }
 
-    if [ -f /etc/ssl/certs/ca-certificates.crt ]; then
-      content=$(</etc/ssl/certs/ca-certificates.crt)
-      if [ "$content" == CERT_CONTENT ]; then
-        report present
+      if [ -f /etc/ssl/certs/ca-certificates.crt ]; then
+        content=$(</etc/ssl/certs/ca-certificates.crt)
+        if [ "$content" == CERT_CONTENT ]; then
+          report present
+        fi
+      else
+        report missing
       fi
-    else
-      report missing
-    fi
 
-    # Always fail, because we do not want to bother with fixed-output
-    # derivations being cached, and do not want to compute the right hash.
-    false;
-  '';
-} // (
-  if fixed-output == "fixed-output"
-  then { outputHash = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; }
-  else { }
-))
+      # Always fail, because we do not want to bother with fixed-output
+      # derivations being cached, and do not want to compute the right hash.
+      false;
+    '';
+  } // {
+    fixed-output = { outputHash = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; };
+    normal = { };
+  }.${mode}
+)
 
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 45f0ce7a4..5c4e0cae9 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -34,19 +34,23 @@ nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
 # Test that sandboxed builds with --check and -K can move .check directory to store
 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
 
-(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
-if grepQuiet 'error: renaming' $TEST_ROOT/log; then false; fi
-grepQuiet 'may not be deterministic' $TEST_ROOT/log
+expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \
+  | tee >( grepQuietInverse 'error: renaming' ) \
+  | grepQuiet 'may not be deterministic'
 
 # Test that sandboxed builds cannot write to /etc easily
-(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store)
+expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store
 
 
 ## Test mounting of SSL certificates into the sandbox
 testCert () {
-    (! nix-build linux-sandbox-cert-test.nix --argstr fixed-output "$2" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$3" 2> $TEST_ROOT/log)
-    cat $TEST_ROOT/log
-    grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log
+    expectation=$1 # "missing" | "present"
+    mode=$2        # "normal" | "fixed-output"
+    certFile=$3    # a string that can be the path to a cert file
+    [ "$mode" == fixed-output ] && ret=1 || ret=100
+    expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \
+      # tee /dev/stderr | \
+      grepQuiet "CERT_${expectation}_IN_SANDBOX"
 }
 
 nocert=$TEST_ROOT/no-cert-file.pem

From a420ccc6a8d1ceee8a413f96c61af0c968b1c255 Mon Sep 17 00:00:00 2001
From: Peter Becich <peterbecich@gmail.com>
Date: Mon, 22 May 2023 21:59:44 -0700
Subject: [PATCH 031/183] `nix flake check`: skip derivations for foreign
 systems (#7759)

`nix flake show` now skips derivations for foreign systems: https://github.com/NixOS/nix/pull/6988

This commit borrows from that to implement the same behavior for `nix flake check`.

See "nix flake check breaks on IFD in multi-platform flake" https://github.com/NixOS/nix/issues/4265
---
 src/nix/flake.cc      | 121 ++++++++++++++++++++++++++++--------------
 tests/flakes/check.sh |   4 +-
 2 files changed, 85 insertions(+), 40 deletions(-)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 3db655aeb..1eea52e15 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -259,6 +259,7 @@ struct CmdFlakeInfo : CmdFlakeMetadata
 struct CmdFlakeCheck : FlakeCommand
 {
     bool build = true;
+    bool checkAllSystems = false;
 
     CmdFlakeCheck()
     {
@@ -267,6 +268,11 @@ struct CmdFlakeCheck : FlakeCommand
             .description = "Do not build checks.",
             .handler = {&build, false}
         });
+        addFlag({
+            .longName = "all-systems",
+            .description = "Check the outputs for all systems.",
+            .handler = {&checkAllSystems, true}
+        });
     }
 
     std::string description() override
@@ -292,6 +298,7 @@ struct CmdFlakeCheck : FlakeCommand
 
         lockFlags.applyNixConfig = true;
         auto flake = lockFlake();
+        auto localSystem = std::string(settings.thisSystem.get());
 
         bool hasErrors = false;
         auto reportError = [&](const Error & e) {
@@ -307,6 +314,8 @@ struct CmdFlakeCheck : FlakeCommand
             }
         };
 
+        std::set<std::string> omittedSystems;
+
         // FIXME: rewrite to use EvalCache.
 
         auto resolve = [&] (PosIdx p) {
@@ -327,6 +336,15 @@ struct CmdFlakeCheck : FlakeCommand
                 reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos)));
         };
 
+        auto checkSystemType = [&](const std::string & system, const PosIdx pos) {
+            if (!checkAllSystems && system != localSystem) {
+                omittedSystems.insert(system);
+                return false;
+            } else {
+                return true;
+            }
+        };
+
         auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional<StorePath> {
             try {
                 auto drvInfo = getDerivation(*state, v, false);
@@ -509,16 +527,18 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                state->forceAttrs(*attr.value, attr.pos, "");
-                                for (auto & attr2 : *attr.value->attrs) {
-                                    auto drvPath = checkDerivation(
-                                        fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
-                                        *attr2.value, attr2.pos);
-                                    if (drvPath && attr_name == settings.thisSystem.get()) {
-                                        drvPaths.push_back(DerivedPath::Built {
-                                            .drvPath = *drvPath,
-                                            .outputs = OutputsSpec::All { },
-                                        });
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    state->forceAttrs(*attr.value, attr.pos, "");
+                                    for (auto & attr2 : *attr.value->attrs) {
+                                        auto drvPath = checkDerivation(
+                                            fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
+                                            *attr2.value, attr2.pos);
+                                        if (drvPath && attr_name == settings.thisSystem.get()) {
+                                            drvPaths.push_back(DerivedPath::Built {
+                                                    .drvPath = *drvPath,
+                                                        .outputs = OutputsSpec::All { },
+                                                        });
+                                        }
                                     }
                                 }
                             }
@@ -529,9 +549,11 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                checkApp(
-                                    fmt("%s.%s", name, attr_name),
-                                    *attr.value, attr.pos);
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    checkApp(
+                                        fmt("%s.%s", name, attr_name),
+                                        *attr.value, attr.pos);
+                                };
                             }
                         }
 
@@ -540,11 +562,13 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                state->forceAttrs(*attr.value, attr.pos, "");
-                                for (auto & attr2 : *attr.value->attrs)
-                                    checkDerivation(
-                                        fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
-                                        *attr2.value, attr2.pos);
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    state->forceAttrs(*attr.value, attr.pos, "");
+                                    for (auto & attr2 : *attr.value->attrs)
+                                        checkDerivation(
+                                            fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
+                                            *attr2.value, attr2.pos);
+                                };
                             }
                         }
 
@@ -553,11 +577,13 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                state->forceAttrs(*attr.value, attr.pos, "");
-                                for (auto & attr2 : *attr.value->attrs)
-                                    checkApp(
-                                        fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
-                                        *attr2.value, attr2.pos);
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    state->forceAttrs(*attr.value, attr.pos, "");
+                                    for (auto & attr2 : *attr.value->attrs)
+                                        checkApp(
+                                            fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
+                                            *attr2.value, attr2.pos);
+                                };
                             }
                         }
 
@@ -566,9 +592,11 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                checkDerivation(
-                                    fmt("%s.%s", name, attr_name),
-                                    *attr.value, attr.pos);
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    checkDerivation(
+                                        fmt("%s.%s", name, attr_name),
+                                        *attr.value, attr.pos);
+                                };
                             }
                         }
 
@@ -577,9 +605,11 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                checkApp(
-                                    fmt("%s.%s", name, attr_name),
-                                    *attr.value, attr.pos);
+                                if (checkSystemType(attr_name, attr.pos) ) {
+                                    checkApp(
+                                        fmt("%s.%s", name, attr_name),
+                                        *attr.value, attr.pos);
+                                };
                             }
                         }
 
@@ -587,6 +617,7 @@ struct CmdFlakeCheck : FlakeCommand
                             state->forceAttrs(vOutput, pos, "");
                             for (auto & attr : *vOutput.attrs) {
                                 checkSystemName(state->symbols[attr.name], attr.pos);
+                                checkSystemType(state->symbols[attr.name], attr.pos);
                                 // FIXME: do getDerivations?
                             }
                         }
@@ -636,9 +667,11 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                checkBundler(
-                                    fmt("%s.%s", name, attr_name),
-                                    *attr.value, attr.pos);
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    checkBundler(
+                                        fmt("%s.%s", name, attr_name),
+                                        *attr.value, attr.pos);
+                                };
                             }
                         }
 
@@ -647,12 +680,14 @@ struct CmdFlakeCheck : FlakeCommand
                             for (auto & attr : *vOutput.attrs) {
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
-                                state->forceAttrs(*attr.value, attr.pos, "");
-                                for (auto & attr2 : *attr.value->attrs) {
-                                    checkBundler(
-                                        fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
-                                        *attr2.value, attr2.pos);
-                                }
+                                if (checkSystemType(attr_name, attr.pos)) {
+                                    state->forceAttrs(*attr.value, attr.pos, "");
+                                    for (auto & attr2 : *attr.value->attrs) {
+                                        checkBundler(
+                                            fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]),
+                                            *attr2.value, attr2.pos);
+                                    }
+                                };
                             }
                         }
 
@@ -685,7 +720,15 @@ struct CmdFlakeCheck : FlakeCommand
         }
         if (hasErrors)
             throw Error("some errors were encountered during the evaluation");
-    }
+
+        if (!omittedSystems.empty()) {
+            warn(
+                "The check omitted these incompatible systems: %s\n"
+                "Use '--all-systems' to check all.",
+                concatStringsSep(", ", omittedSystems)
+            );
+        };
+    };
 };
 
 static Strings defaultTemplateAttrPathsPrefixes{"templates."};
diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh
index 865ca61b4..34b82c61c 100644
--- a/tests/flakes/check.sh
+++ b/tests/flakes/check.sh
@@ -72,6 +72,8 @@ cat > $flakeDir/flake.nix <<EOF
 }
 EOF
 
-checkRes=$(nix flake check --keep-going $flakeDir 2>&1 && fail "nix flake check should have failed" || true)
+nix flake check $flakeDir
+
+checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true)
 echo "$checkRes" | grepQuiet "packages.system-1.default"
 echo "$checkRes" | grepQuiet "packages.system-2.default"

From f0233f3a3fd92d443d2502ed29aea076b6572ab7 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Tue, 23 May 2023 09:59:49 +0200
Subject: [PATCH 032/183] Further refactor linux-sandbox.sh and fix tee usage

---
 tests/linux-sandbox.sh | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 5c4e0cae9..19a7527bf 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -11,6 +11,8 @@ requireSandboxSupport
 # otherwise things get complicated (e.g. if it's in /bin, do we need
 # /lib as well?).
 if [[ ! $SHELL =~ /nix/store ]]; then skipTest "Shell is not from Nix store"; fi
+# An alias to automatically bind-mount the $SHELL on nix-build invocations
+nix-sandbox-build () { nix-build --no-out-link --sandbox-paths /nix/store "$@"; }
 
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0
@@ -18,7 +20,7 @@ rm -rf $TEST_ROOT/store0
 export NIX_STORE_DIR=/my/store
 export NIX_REMOTE=$TEST_ROOT/store0
 
-outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store)
+outPath=$(nix-sandbox-build dependencies.nix)
 
 [[ $outPath =~ /my/store/.*-dependencies ]]
 
@@ -29,17 +31,18 @@ nix store ls -R -l $outPath | grep foobar
 nix store cat $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
-nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
+nix-sandbox-build dependencies.nix --check
 
 # Test that sandboxed builds with --check and -K can move .check directory to store
-nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
+nix-sandbox-build check.nix -A nondeterministic
 
-expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \
-  | tee >( grepQuietInverse 'error: renaming' ) \
-  | grepQuiet 'may not be deterministic'
+expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log
+grepQuietInverse 'error: renaming' $TEST_ROOT/log
+grepQuiet 'may not be deterministic' $TEST_ROOT/log
 
 # Test that sandboxed builds cannot write to /etc easily
-expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store
+expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' |
+    grepQuiet "/etc/test: Permission denied"
 
 
 ## Test mounting of SSL certificates into the sandbox
@@ -48,9 +51,9 @@ testCert () {
     mode=$2        # "normal" | "fixed-output"
     certFile=$3    # a string that can be the path to a cert file
     [ "$mode" == fixed-output ] && ret=1 || ret=100
-    expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \
-      # tee /dev/stderr | \
-      grepQuiet "CERT_${expectation}_IN_SANDBOX"
+    expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" |
+        # tee /dev/stderr |
+        grepQuiet "CERT_${expectation}_IN_SANDBOX"
 }
 
 nocert=$TEST_ROOT/no-cert-file.pem

From 303858afad8f4084c8b15d66ba02ee083452bb73 Mon Sep 17 00:00:00 2001
From: oxalica <oxalicc@pm.me>
Date: Wed, 24 May 2023 01:12:02 +0800
Subject: [PATCH 033/183] Fix typo in error message of too long store path

---
 src/libstore/path.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 46be54281..552e83114 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -9,8 +9,8 @@ static void checkName(std::string_view path, std::string_view name)
     if (name.empty())
         throw BadStorePath("store path '%s' has an empty name", path);
     if (name.size() > StorePath::MaxPathLen)
-        throw BadStorePath("store path '%s' has a name longer than '%d characters",
-            StorePath::MaxPathLen, path);
+        throw BadStorePath("store path '%s' has a name longer than %d characters",
+            path, StorePath::MaxPathLen);
     // See nameRegexStr for the definition
     for (auto c : name)
         if (!((c >= '0' && c <= '9')

From 3ebe1341abe1b0ad59bd4925517af18d9200f818 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 17 Mar 2023 15:51:08 +0100
Subject: [PATCH 034/183] Make `RewritingSink` accept a map of rewrites

Giving it the same semantics as `rewriteStrings`.
Also add some tests for it
---
 src/libexpr/primops.cc                      |  2 +-
 src/libstore/build/local-derivation-goal.cc |  2 +-
 src/libstore/path-references.cc             | 73 +++++++++++++++++
 src/libstore/path-references.hh             | 25 ++++++
 src/{libstore => libutil}/references.cc     | 86 +++++----------------
 src/{libstore => libutil}/references.hh     | 23 +-----
 src/libutil/tests/references.cc             | 46 +++++++++++
 7 files changed, 169 insertions(+), 88 deletions(-)
 create mode 100644 src/libstore/path-references.cc
 create mode 100644 src/libstore/path-references.hh
 rename src/{libstore => libutil}/references.cc (61%)
 rename src/{libstore => libutil}/references.hh (61%)
 create mode 100644 src/libutil/tests/references.cc

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cfae1e5f8..572e76ec6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -6,7 +6,7 @@
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
-#include "references.hh"
+#include "path-references.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "value-to-json.hh"
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 05d6685da..6f7ab8a3d 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -4,7 +4,7 @@
 #include "worker.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
-#include "references.hh"
+#include "path-references.hh"
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc
new file mode 100644
index 000000000..33cf66ce3
--- /dev/null
+++ b/src/libstore/path-references.cc
@@ -0,0 +1,73 @@
+#include "path-references.hh"
+#include "hash.hh"
+#include "util.hh"
+#include "archive.hh"
+
+#include <map>
+#include <cstdlib>
+#include <mutex>
+#include <algorithm>
+
+
+namespace nix {
+
+
+PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
+    : RefScanSink(std::move(hashes))
+    , backMap(std::move(backMap))
+{ }
+
+PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
+{
+    StringSet hashes;
+    std::map<std::string, StorePath> backMap;
+
+    for (auto & i : refs) {
+        std::string hashPart(i.hashPart());
+        auto inserted = backMap.emplace(hashPart, i).second;
+        assert(inserted);
+        hashes.insert(hashPart);
+    }
+
+    return PathRefScanSink(std::move(hashes), std::move(backMap));
+}
+
+StorePathSet PathRefScanSink::getResultPaths()
+{
+    /* Map the hashes found back to their store paths. */
+    StorePathSet found;
+    for (auto & i : getResult()) {
+        auto j = backMap.find(i);
+        assert(j != backMap.end());
+        found.insert(j->second);
+    }
+
+    return found;
+}
+
+
+std::pair<StorePathSet, HashResult> scanForReferences(
+    const std::string & path,
+    const StorePathSet & refs)
+{
+    HashSink hashSink { htSHA256 };
+    auto found = scanForReferences(hashSink, path, refs);
+    auto hash = hashSink.finish();
+    return std::pair<StorePathSet, HashResult>(found, hash);
+}
+
+StorePathSet scanForReferences(
+    Sink & toTee,
+    const Path & path,
+    const StorePathSet & refs)
+{
+    PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
+    TeeSink sink { refsSink, toTee };
+
+    /* Look for the hashes in the NAR dump of the path. */
+    dumpPath(path, sink);
+
+    return refsSink.getResultPaths();
+}
+
+}
diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh
new file mode 100644
index 000000000..7b44e3261
--- /dev/null
+++ b/src/libstore/path-references.hh
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "references.hh"
+#include "path.hh"
+
+namespace nix {
+
+std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
+
+StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
+
+class PathRefScanSink : public RefScanSink
+{
+    std::map<std::string, StorePath> backMap;
+
+    PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
+
+public:
+
+    static PathRefScanSink fromPaths(const StorePathSet & refs);
+
+    StorePathSet getResultPaths();
+};
+
+}
diff --git a/src/libstore/references.cc b/src/libutil/references.cc
similarity index 61%
rename from src/libstore/references.cc
rename to src/libutil/references.cc
index 345f4528b..74003584a 100644
--- a/src/libstore/references.cc
+++ b/src/libutil/references.cc
@@ -6,6 +6,7 @@
 #include <map>
 #include <cstdlib>
 #include <mutex>
+#include <algorithm>
 
 
 namespace nix {
@@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data)
 }
 
 
-PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
-    : RefScanSink(std::move(hashes))
-    , backMap(std::move(backMap))
-{ }
-
-PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
-{
-    StringSet hashes;
-    std::map<std::string, StorePath> backMap;
-
-    for (auto & i : refs) {
-        std::string hashPart(i.hashPart());
-        auto inserted = backMap.emplace(hashPart, i).second;
-        assert(inserted);
-        hashes.insert(hashPart);
-    }
-
-    return PathRefScanSink(std::move(hashes), std::move(backMap));
-}
-
-StorePathSet PathRefScanSink::getResultPaths()
-{
-    /* Map the hashes found back to their store paths. */
-    StorePathSet found;
-    for (auto & i : getResult()) {
-        auto j = backMap.find(i);
-        assert(j != backMap.end());
-        found.insert(j->second);
-    }
-
-    return found;
-}
-
-
-std::pair<StorePathSet, HashResult> scanForReferences(
-    const std::string & path,
-    const StorePathSet & refs)
-{
-    HashSink hashSink { htSHA256 };
-    auto found = scanForReferences(hashSink, path, refs);
-    auto hash = hashSink.finish();
-    return std::pair<StorePathSet, HashResult>(found, hash);
-}
-
-StorePathSet scanForReferences(
-    Sink & toTee,
-    const Path & path,
-    const StorePathSet & refs)
-{
-    PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
-    TeeSink sink { refsSink, toTee };
-
-    /* Look for the hashes in the NAR dump of the path. */
-    dumpPath(path, sink);
-
-    return refsSink.getResultPaths();
-}
-
-
 RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
-    : from(from), to(to), nextSink(nextSink)
+    : RewritingSink({{from, to}}, nextSink)
 {
-    assert(from.size() == to.size());
+}
+
+RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink)
+    : rewrites(rewrites), nextSink(nextSink)
+{
+    long unsigned int maxRewriteSize = 0;
+    for (auto & [from, to] : rewrites) {
+        assert(from.size() == to.size());
+        maxRewriteSize = std::max(maxRewriteSize, from.size());
+    }
+    this->maxRewriteSize = maxRewriteSize;
 }
 
 void RewritingSink::operator () (std::string_view data)
@@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data)
     std::string s(prev);
     s.append(data);
 
-    size_t j = 0;
-    while ((j = s.find(from, j)) != std::string::npos) {
-        matches.push_back(pos + j);
-        s.replace(j, from.size(), to);
-    }
+    s = rewriteStrings(s, rewrites);
 
-    prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
+    prev = s.size() < maxRewriteSize
+        ? s
+        : maxRewriteSize == 0
+            ? ""
+            : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1);
 
     auto consumed = s.size() - prev.size();
 
diff --git a/src/libstore/references.hh b/src/libutil/references.hh
similarity index 61%
rename from src/libstore/references.hh
rename to src/libutil/references.hh
index 52d71b333..ffd730e7b 100644
--- a/src/libstore/references.hh
+++ b/src/libutil/references.hh
@@ -2,14 +2,9 @@
 ///@file
 
 #include "hash.hh"
-#include "path.hh"
 
 namespace nix {
 
-std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
-
-StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
-
 class RefScanSink : public Sink
 {
     StringSet hashes;
@@ -28,28 +23,18 @@ public:
     void operator () (std::string_view data) override;
 };
 
-class PathRefScanSink : public RefScanSink
-{
-    std::map<std::string, StorePath> backMap;
-
-    PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
-
-public:
-
-    static PathRefScanSink fromPaths(const StorePathSet & refs);
-
-    StorePathSet getResultPaths();
-};
-
 struct RewritingSink : Sink
 {
-    std::string from, to, prev;
+    const StringMap rewrites;
+    long unsigned int maxRewriteSize;
+    std::string prev;
     Sink & nextSink;
     uint64_t pos = 0;
 
     std::vector<uint64_t> matches;
 
     RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
+    RewritingSink(const StringMap & rewrites, Sink & nextSink);
 
     void operator () (std::string_view data) override;
 
diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc
new file mode 100644
index 000000000..a517d9aa1
--- /dev/null
+++ b/src/libutil/tests/references.cc
@@ -0,0 +1,46 @@
+#include "references.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+using std::string;
+
+struct RewriteParams {
+    string originalString, finalString;
+    StringMap rewrites;
+
+    friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) {
+        StringSet strRewrites;
+        for (auto & [from, to] : bar.rewrites)
+            strRewrites.insert(from + "->" + to);
+        return os <<
+            "OriginalString: " << bar.originalString << std::endl <<
+            "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl <<
+            "Expected result: " << bar.finalString;
+    }
+};
+
+class RewriteTest : public ::testing::TestWithParam<RewriteParams> {
+};
+
+TEST_P(RewriteTest, IdentityRewriteIsIdentity) {
+    RewriteParams param = GetParam();
+    StringSink rewritten;
+    auto rewriter = RewritingSink(param.rewrites, rewritten);
+    rewriter(param.originalString);
+    rewriter.flush();
+    ASSERT_EQ(rewritten.s, param.finalString);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    references,
+    RewriteTest,
+    ::testing::Values(
+        RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}},
+        RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}},
+        RewriteParams{ "foooo", "foooo", {}}
+    )
+);
+
+}
+

From a917fb0d53544f7fd29f24d5984702296d457347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 17 Mar 2023 15:51:08 +0100
Subject: [PATCH 035/183] Use a RewritingSink in derivation goal

Possibly this will make it stream
---
 src/libstore/build/local-derivation-goal.cc | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 6f7ab8a3d..4650b8bae 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2384,13 +2384,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             if (!outputRewrites.empty()) {
                 debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
-                /* FIXME: this is in-memory. */
-                StringSink sink;
-                dumpPath(actualPath, sink);
+                /* FIXME: Is this actually streaming? */
+                auto source = sinkToSource([&](Sink & nextSink) {
+                    RewritingSink rsink(outputRewrites, nextSink);
+                    dumpPath(actualPath, rsink);
+                    rsink.flush();
+                });
+                Path tmpPath = actualPath + ".tmp";
+                restorePath(tmpPath, *source);
                 deletePath(actualPath);
-                sink.s = rewriteStrings(sink.s, outputRewrites);
-                StringSource source(sink.s);
-                restorePath(actualPath, source);
+                movePath(tmpPath, actualPath);
 
                 /* FIXME: set proper permissions in restorePath() so
                    we don't have to do another traversal. */

From 34e1b464f0054623334355aac94cb1a3e4c35a96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 17 Mar 2023 15:51:08 +0100
Subject: [PATCH 036/183] Normalize the hash-rewriting process when building
 derivations

---
 src/libstore/build/local-derivation-goal.cc | 28 ++++++++++-----------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 4650b8bae..e9fe1eb73 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2379,14 +2379,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             continue;
         auto references = *referencesOpt;
 
-        auto rewriteOutput = [&]() {
+        auto rewriteOutput = [&](const StringMap & rewrites) {
             /* Apply hash rewriting if necessary. */
-            if (!outputRewrites.empty()) {
+            if (!rewrites.empty()) {
                 debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
                 /* FIXME: Is this actually streaming? */
                 auto source = sinkToSource([&](Sink & nextSink) {
-                    RewritingSink rsink(outputRewrites, nextSink);
+                    RewritingSink rsink(rewrites, nextSink);
                     dumpPath(actualPath, rsink);
                     rsink.flush();
                 });
@@ -2442,7 +2442,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                         "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
                         actualPath);
             }
-            rewriteOutput();
+            rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
             HashModuloSink caSink { outputHash.hashType, oldHashPart };
@@ -2480,16 +2480,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 Hash::dummy,
             };
             if (*scratchPath != newInfo0.path) {
-                // Also rewrite the output path
-                auto source = sinkToSource([&](Sink & nextSink) {
-                    RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
-                    dumpPath(actualPath, rsink2);
-                    rsink2.flush();
-                });
-                Path tmpPath = actualPath + ".tmp";
-                restorePath(tmpPath, *source);
-                deletePath(actualPath);
-                movePath(tmpPath, actualPath);
+                // If the path has some self-references, we need to rewrite
+                // them.
+                // (note that this doesn't invalidate the ca hash we calculated
+                // above because it's computed *modulo the self-references*, so
+                // it already takes this rewrite into account).
+                rewriteOutput(
+                    StringMap{{oldHashPart,
+                               std::string(newInfo0.path.hashPart())}});
             }
 
             HashResult narHashAndSize = hashPath(htSHA256, actualPath);
@@ -2511,7 +2509,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                     outputRewrites.insert_or_assign(
                         std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
-                rewriteOutput();
+                rewriteOutput(outputRewrites);
                 auto narHashAndSize = hashPath(htSHA256, actualPath);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;

From d0cecbe87708bd61548afa51fea78ec927403c5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Wed, 24 May 2023 15:35:46 +0200
Subject: [PATCH 037/183] Disable the fetchClosure test for old daemons

Broken because of the change introduced by #4282
---
 tests/fetchClosure.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh
index a207f647c..21650eb06 100644
--- a/tests/fetchClosure.sh
+++ b/tests/fetchClosure.sh
@@ -5,6 +5,12 @@ enableFeatures "fetch-closure"
 clearStore
 clearCacheCache
 
+# Old daemons don't properly zero out the self-references when
+# calculating the CA hashes, so this breaks `nix store
+# make-content-addressed` which expects the client and the daemon to
+# compute the same hash
+requireDaemonNewerThan "2.16.0pre20230524"
+
 # Initialize binary cache.
 nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out)
 caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')

From c694f1a2f3773664b08e561c3fe75b5fe97df51f Mon Sep 17 00:00:00 2001
From: Aneesh Agrawal <aneeshusa@gmail.com>
Date: Wed, 24 May 2023 11:05:40 -0400
Subject: [PATCH 038/183] Remove old default from docs for `hashed-mirrors`

The `hashed-mirrors` option did use to have this default value,
but it was removed and re-added with an empty default value.
As the autogenerated docs show the (actual) default values from code,
remove this incorrect reference from the docs.

I was updating my nix.conf settings after a few years and noticed this.
---
 src/libstore/globals.hh | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 31dfe5b4e..6d8a0af32 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -896,12 +896,11 @@ public:
         this, {}, "hashed-mirrors",
         R"(
           A list of web servers used by `builtins.fetchurl` to obtain files by
-          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
-          *ht* and a base-16 hash *h*, Nix will try to download the file from
-          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
-          they have disappeared from their original URI. For example, given
-          the default mirror `http://tarballs.nixos.org/`, when building the
-          derivation
+          hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
+          download the file from *hashed-mirror*/*ht*/*h*. This allows files to
+          be downloaded even if they have disappeared from their original URI.
+          For example, given an example mirror `http://tarballs.nixos.org/`,
+          when building the derivation
 
           ```nix
           builtins.fetchurl {

From d16a1994fb6048d4ea48090c5aabafb7ad89c84f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Thu, 25 May 2023 16:38:29 +0200
Subject: [PATCH 039/183] Properly report build errors on chrooted stores
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When encountering a build error, Nix moves the output paths out of the
chroot into their final location (for “easier debugging of build
failures”). However this was broken for chroot stores as it was moving
it to the _logical_ location, not the _physical_ one.

Fix it by moving to the physical (_real_) location.

Fix https://github.com/NixOS/nix/issues/8395
---
 src/libstore/build/local-derivation-goal.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 05d6685da..b0289ac75 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         for (auto & [_, status] : initialOutputs) {
             if (!status.known) continue;
             if (buildMode != bmCheck && status.known->isValid()) continue;
-            auto p = worker.store.printStorePath(status.known->path);
+            auto p = worker.store.toRealPath(status.known->path);
             if (pathExists(chrootRootDir + p))
                 renameFile((chrootRootDir + p), p);
         }

From 367fcd9eb590d002cbbae23f22b1395b7d3303e6 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Thu, 2 Mar 2023 17:29:36 -0700
Subject: [PATCH 040/183] installer: don't assume GNU diff

macOS Ventura ships with it's own version of diff. Try to output a
similar diff with Apple diff as with GNU diff, instead of failing

Helps https://github.com/NixOS/nix/issues/7286
---
 scripts/install-multi-user.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 7c66538b0..79deb2819 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -246,8 +246,15 @@ printf -v _OLD_LINE_FMT "%b" $'\033[1;7;31m-'"$ESC ${RED}%L${ESC}"
 printf -v _NEW_LINE_FMT "%b" $'\033[1;7;32m+'"$ESC ${GREEN}%L${ESC}"
 
 _diff() {
+    # macOS Ventura doesn't ship with GNU diff. Print similar output except
+    # without +/- markers or dimming
+    if diff --version | grep -q "Apple diff"; then
+        printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}"
+        diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@"
+    else
     # simple colorized diff comatible w/ pre `--color` versions
-    diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format="  %L" "$@"
+        diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format="  %L" "$@"
+    fi
 }
 
 confirm_rm() {

From a382919d7dd696c01c0d5abd04222c2821c0a49d Mon Sep 17 00:00:00 2001
From: polykernel <81340136+polykernel@users.noreply.github.com>
Date: Thu, 25 May 2023 00:37:00 -0400
Subject: [PATCH 041/183] primops: lazy evaluation of replaceStrings
 replacements

The primop `builtins.replaceStrings` currently always strictly evaluates the
replacement strings, however time and space are wasted for their computation
if the corresponding pattern do not occur in the input string. This commit
makes the evaluation of the replacement strings lazy by deferring their
evaluation to when the corresponding pattern are matched and memoize the result
for efficient retrieval on subsequent matches.

The testcases for replaceStrings was updated to check for lazy evaluation
of the replacements. A note was also added in the release notes to
document the behavior change.
---
 doc/manual/src/release-notes/rl-next.md |  1 +
 src/libexpr/primops.cc                  | 25 +++++++++++++------------
 src/libexpr/tests/error_traces.cc       |  2 +-
 tests/lang/eval-okay-replacestrings.exp |  2 +-
 tests/lang/eval-okay-replacestrings.nix |  1 +
 5 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index bc0d41bdf..f92625887 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -4,3 +4,4 @@
   The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
   The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
   The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).
+- The function `builtins.replaceStrings` is now lazy in the value of its second argument `to`, that is a replacee in `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cfae1e5f8..0f3b0a9fe 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3910,13 +3910,8 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
     for (auto elem : args[0]->listItems())
         from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-    std::vector<std::pair<std::string, NixStringContext>> to;
-    to.reserve(args[1]->listSize());
-    for (auto elem : args[1]->listItems()) {
-        NixStringContext ctx;
-        auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
-        to.emplace_back(s, std::move(ctx));
-    }
+    std::unordered_map<size_t, std::string> cache;
+    auto to = args[1]->listItems();
 
     NixStringContext context;
     auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings");
@@ -3927,10 +3922,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
-        for (; i != from.end(); ++i, ++j)
+        size_t j_index = 0;
+        for (; i != from.end(); ++i, ++j, ++j_index)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
-                res += j->first;
+                auto v = cache.find(j_index);
+                if (v == cache.end()) {
+                    NixStringContext ctx;
+                    auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
+                    v = (cache.emplace(j_index, ts)).first;
+                    for (auto& path : ctx)
+                        context.insert(path);
+                }
+                res += v->second;
                 if (i->empty()) {
                     if (p < s.size())
                         res += s[p];
@@ -3938,9 +3942,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
                 } else {
                     p += i->size();
                 }
-                for (auto& path : j->second)
-                    context.insert(path);
-                j->second.clear();
                 break;
             }
         if (!found) {
diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc
index 24e95ac39..285651256 100644
--- a/src/libexpr/tests/error_traces.cc
+++ b/src/libexpr/tests/error_traces.cc
@@ -171,7 +171,7 @@ namespace nix {
                       hintfmt("value is %s while a string was expected", "an integer"),
                       hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-        ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}",
+        ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"",
                       TypeError,
                       hintfmt("value is %s while a string was expected", "a Boolean"),
                       hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings"));
diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp
index 72e8274d8..eac67c5fe 100644
--- a/tests/lang/eval-okay-replacestrings.exp
+++ b/tests/lang/eval-okay-replacestrings.exp
@@ -1 +1 @@
-[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ]
+[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ]
diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix
index bd8031fc0..a803e6519 100644
--- a/tests/lang/eval-okay-replacestrings.nix
+++ b/tests/lang/eval-okay-replacestrings.nix
@@ -8,4 +8,5 @@ with builtins;
   (replaceStrings [""] ["X"] "abc")
   (replaceStrings [""] ["X"] "")
   (replaceStrings ["-"] ["_"] "a-b")
+  (replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar")
 ]

From be4890747051de0e489d608fdba65489c45d2b02 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Fri, 17 Feb 2023 19:23:09 +0100
Subject: [PATCH 042/183] ci: Always run with sandbox, even on Darwin

And fix a test failure in the sandbox due to /home
existing on Darwin but not being accessible in the sandbox since it's a
symlink to /System/Volumes/Data/home, see
https://github.com/NixOS/nix/actions/runs/4205378453/jobs/7297384658#step:6:2127:

    C++ exception with description "error: getting status of /home/schnitzel/darmstadt/pommes: Operation not permitted" thrown in the test body.

On Linux this wasn't a problem because there /home doesn't exist in the sandbox
---
 .github/workflows/ci.yml   | 3 +++
 src/libutil/tests/tests.cc | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c06c77043..61e7f4560 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,6 +20,9 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v20
+      with:
+        # The sandbox would otherwise be disabled by default on Darwin
+        extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 250e83a38..f3c1e8248 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -202,7 +202,7 @@ namespace nix {
     }
 
     TEST(pathExists, bogusPathDoesNotExist) {
-        ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes"));
+        ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes"));
     }
 
     /* ----------------------------------------------------------------------------

From b7cde90c6b479562eb9f1d9df399d04cf9c42aad Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Thu, 11 May 2023 13:09:02 +0200
Subject: [PATCH 043/183] Call getDefaultSSLCertFile() only when none is
 specified

This does pathExists on various paths, which crashes on EPERM in the
macOS sandbox.
---
 src/libstore/globals.cc | 2 ++
 src/libstore/globals.hh | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4c66d08ee..a196c10e6 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -57,6 +57,8 @@ Settings::Settings()
     auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (sslOverride != "")
         caFile = sslOverride;
+    else if (caFile == "")
+        caFile = getDefaultSSLCertFile();
 
     /* Backwards compatibility. */
     auto s = getEnv("NIX_REMOTE_SYSTEMS");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 31dfe5b4e..34b4f24a7 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -842,7 +842,7 @@ public:
         )"};
 
     Setting<Path> caFile{
-        this, getDefaultSSLCertFile(), "ssl-cert-file",
+        this, "", "ssl-cert-file",
         R"(
           The path of a file containing CA certificates used to
           authenticate `https://` downloads. Nix by default will use

From a6c78ba367725a81aa631a7df2d0840ddd25faf5 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Thu, 11 May 2023 13:44:16 +0200
Subject: [PATCH 044/183] getDefaultNixPath: ignore EPERM

---
 src/libexpr/eval.cc | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 585670e69..61012f2ab 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2620,12 +2620,17 @@ Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
     auto add = [&](const Path & p, const std::string & s = std::string()) {
-        if (pathExists(p)) {
-            if (s.empty()) {
-                res.push_back(p);
-            } else {
-                res.push_back(s + "=" + p);
+        try {
+            if (pathExists(p)) {
+                if (s.empty()) {
+                    res.push_back(p);
+                } else {
+                    res.push_back(s + "=" + p);
+                }
             }
+        } catch (SysError & e) {
+            // swallow EPERM
+            if (e.errNo != EPERM) throw;
         }
     };
 

From 2c462486fe0c1bcb5b1142507d2875e98b2418df Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Fri, 26 May 2023 15:32:28 +0200
Subject: [PATCH 045/183] create pathAccessible, use it to infer default dirs

---
 src/libexpr/eval.cc     | 15 +++++----------
 src/libstore/globals.cc |  4 +---
 src/libstore/globals.hh |  2 +-
 src/libutil/util.cc     | 11 +++++++++++
 src/libutil/util.hh     |  8 ++++++++
 5 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 61012f2ab..71fd6e6e4 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2620,17 +2620,12 @@ Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
     auto add = [&](const Path & p, const std::string & s = std::string()) {
-        try {
-            if (pathExists(p)) {
-                if (s.empty()) {
-                    res.push_back(p);
-                } else {
-                    res.push_back(s + "=" + p);
-                }
+        if (pathAccessible(p)) {
+            if (s.empty()) {
+                res.push_back(p);
+            } else {
+                res.push_back(s + "=" + p);
             }
-        } catch (SysError & e) {
-            // swallow EPERM
-            if (e.errNo != EPERM) throw;
         }
     };
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index a196c10e6..32e9a6ea9 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -57,8 +57,6 @@ Settings::Settings()
     auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (sslOverride != "")
         caFile = sslOverride;
-    else if (caFile == "")
-        caFile = getDefaultSSLCertFile();
 
     /* Backwards compatibility. */
     auto s = getEnv("NIX_REMOTE_SYSTEMS");
@@ -185,7 +183,7 @@ bool Settings::isWSL1()
 Path Settings::getDefaultSSLCertFile()
 {
     for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
-        if (pathExists(fn)) return fn;
+        if (pathAccessible(fn)) return fn;
     return "";
 }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 34b4f24a7..31dfe5b4e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -842,7 +842,7 @@ public:
         )"};
 
     Setting<Path> caFile{
-        this, "", "ssl-cert-file",
+        this, getDefaultSSLCertFile(), "ssl-cert-file",
         R"(
           The path of a file containing CA certificates used to
           authenticate `https://` downloads. Nix by default will use
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 3a8309149..aa0a154fd 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -266,6 +266,17 @@ bool pathExists(const Path & path)
     return false;
 }
 
+bool pathAccessible(const Path & path)
+{
+    try {
+        return pathExists(path);
+    } catch (SysError & e) {
+        // swallow EPERM
+        if (e.errNo == EPERM) return false;
+        throw;
+    }
+}
+
 
 Path readLink(const Path & path)
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a7907cd14..00fcb9b79 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -120,6 +120,14 @@ struct stat lstat(const Path & path);
  */
 bool pathExists(const Path & path);
 
+/**
+ * A version of pathExists that returns false on a permission error.
+ * Useful for inferring default paths across directories that might not
+ * be readable.
+ * @return true iff the given path can be accessed and exists
+ */
+bool pathAccessible(const Path & path);
+
 /**
  * Read the contents (target) of a symbolic link.  The result is not
  * in any way canonicalised.

From e4aebccf209c12714f5547ec5acf08b1fd7f51cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 27 May 2023 17:53:30 +0200
Subject: [PATCH 046/183] Restore Nix 2.3 behaviour for
 {__impure,__contentAddressed} = false

Fixes #8405.
---
 src/libexpr/primops.cc | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cfae1e5f8..341962dc9 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1152,16 +1152,14 @@ drvName, Bindings * attrs, Value & v)
                 if (i->value->type() == nNull) continue;
             }
 
-            if (i->name == state.sContentAddressed) {
-                contentAddressed = state.forceBool(*i->value, noPos, context_below);
-                if (contentAddressed)
-                    experimentalFeatureSettings.require(Xp::CaDerivations);
+            if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) {
+                contentAddressed = true;
+                experimentalFeatureSettings.require(Xp::CaDerivations);
             }
 
-            else if (i->name == state.sImpure) {
-                isImpure = state.forceBool(*i->value, noPos, context_below);
-                if (isImpure)
-                    experimentalFeatureSettings.require(Xp::ImpureDerivations);
+            else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) {
+                isImpure = true;
+                experimentalFeatureSettings.require(Xp::ImpureDerivations);
             }
 
             /* The `args' attribute is special: it supplies the

From 6b56376c89a84388da4f25792c4828baee3cc5e4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 22:56:41 +0000
Subject: [PATCH 047/183] Bump cachix/install-nix-action from 20 to 21

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 20 to 21.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v20...v21)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 61e7f4560..0f1f6d43f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v20
+    - uses: cachix/install-nix-action@v21
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -61,7 +61,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v20
+    - uses: cachix/install-nix-action@v21
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
@@ -82,7 +82,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v20
+    - uses: cachix/install-nix-action@v21
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -109,7 +109,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v20
+    - uses: cachix/install-nix-action@v21
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

From 2e01a53a36737bf9fef6b277d3c8e45827a0c716 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 22:56:44 +0000
Subject: [PATCH 048/183] Bump zeebe-io/backport-action from 1.2.0 to 1.3.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.2.0 to 1.3.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v1.2.0...v1.3.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index b04723b95..37966bab2 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.2.0
+        uses: zeebe-io/backport-action@v1.3.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From f3a88857b46199a44c4389f6e39b59fb9bef70d0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2023 11:49:59 +0200
Subject: [PATCH 049/183] maintainers/upload-release.pl: Don't update
 nix-fallback-paths.nix

---
 maintainers/upload-release.pl | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 77469148a..07b19c13f 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -15,7 +15,6 @@ my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
 my $releasesBucketName = "nix-releases";
 my $channelsBucketName = "nix-channels";
-my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
 my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
 
@@ -200,26 +199,22 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-# Update nix-fallback-paths.nix.
+# Print new nix-fallback-paths.nix.
 if ($isLatest) {
-    system("cd $nixpkgsDir && git pull") == 0 or die;
-
     sub getStorePath {
         my ($jobName) = @_;
         my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
         return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
     }
 
-    write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
+    print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" .
                "{\n" .
                "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
                "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
                "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
                "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
                "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-               "}\n");
-
-    system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
+               "}\n";
 }
 
 # Update the "latest" symlink.

From 7a5731aa0700a1ae7ec3c2d890df235930446b08 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 30 May 2023 22:58:00 +0200
Subject: [PATCH 050/183] link to mentioned builtin

---
 doc/manual/src/release-notes/rl-next.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index f92625887..27eb69cb0 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -4,4 +4,4 @@
   The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
   The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
   The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).
-- The function `builtins.replaceStrings` is now lazy in the value of its second argument `to`, that is a replacee in `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.
+- The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.

From 9afd525e1847650589a376a5d3185241e4936b5d Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 30 May 2023 22:58:15 +0200
Subject: [PATCH 051/183] update documentation according to release notes

---
 src/libexpr/primops.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c28e7cbc7..544785adc 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3957,7 +3957,11 @@ static RegisterPrimOp primop_replaceStrings({
     .args = {"from", "to", "s"},
     .doc = R"(
       Given string *s*, replace every occurrence of the strings in *from*
-      with the corresponding string in *to*. For example,
+      with the corresponding string in *to*.
+
+      The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s*
+
+      Example:
 
       ```nix
       builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"

From 76287a9207ce9b0fbab7ae49f6fa405f0c1ef9ab Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Mon, 29 May 2023 18:17:29 +0300
Subject: [PATCH 052/183] distributed-builds.md: Clarify warning ssh access
 requirements

---
 doc/manual/src/advanced-topics/distributed-builds.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index fefd10100..8f61380e8 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -39,10 +39,11 @@ contains Nix.
 > **Warning**
 >
 > If you are building via the Nix daemon, it is the Nix daemon user
-> account (that is, `root`) that should have SSH access to the remote
+> account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote
 > machine. If you can’t or don’t want to configure `root` to be able to
-> access to remote machine, you can use a private Nix store instead by
-> passing e.g. `--store ~/my-nix`.
+> access the remote machine, you can use a private Nix store instead by
+> passing e.g. `--store ~/my-nix` when running a nix command from the
+> local machine.
 
 The list of remote machines can be specified on the command line or in
 the Nix configuration file. The former is convenient for testing. For

From e2f33e0df10062cfc6d918c9431205cb7c66be4e Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 31 May 2023 01:32:02 +0200
Subject: [PATCH 053/183] rewrap lines to use one line per sentence

this makes future reviews easier as it reduces diff noise
---
 doc/manual/src/advanced-topics/distributed-builds.md | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md
index 8f61380e8..73a113d35 100644
--- a/doc/manual/src/advanced-topics/distributed-builds.md
+++ b/doc/manual/src/advanced-topics/distributed-builds.md
@@ -38,12 +38,9 @@ contains Nix.
 
 > **Warning**
 >
-> If you are building via the Nix daemon, it is the Nix daemon user
-> account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote
-> machine. If you can’t or don’t want to configure `root` to be able to
-> access the remote machine, you can use a private Nix store instead by
-> passing e.g. `--store ~/my-nix` when running a nix command from the
-> local machine.
+> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine.
+>
+> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine.
 
 The list of remote machines can be specified on the command line or in
 the Nix configuration file. The former is convenient for testing. For

From afc24e6a66b5fe3d1fdcc118b12dadda33e08e54 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2023 12:38:05 +0200
Subject: [PATCH 054/183] Release notes

---
 doc/manual/src/SUMMARY.md.in            | 1 +
 doc/manual/src/release-notes/rl-2.16.md | 8 ++++++++
 doc/manual/src/release-notes/rl-next.md | 5 -----
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 doc/manual/src/release-notes/rl-2.16.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 606aecd8f..3149af880 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -105,6 +105,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.16 (2023-05-21)](release-notes/rl-2.16.md)
   - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)
diff --git a/doc/manual/src/release-notes/rl-2.16.md b/doc/manual/src/release-notes/rl-2.16.md
new file mode 100644
index 000000000..97b40d0b8
--- /dev/null
+++ b/doc/manual/src/release-notes/rl-2.16.md
@@ -0,0 +1,8 @@
+# Release 2.16 (2023-05-31)
+
+* Speed-up of downloads from binary caches.
+  The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
+  The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
+  The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).
+
+* The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.
diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 27eb69cb0..78ae99f4b 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,7 +1,2 @@
 # Release X.Y (202?-??-??)
 
-- Speed-up of downloads from binary caches.
-  The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
-  The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
-  The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).
-- The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.

From 008f89fa50fdbdd39de77d48e738f144195cc486 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2023 14:05:00 +0200
Subject: [PATCH 055/183] Typo

---
 doc/manual/src/SUMMARY.md.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 3149af880..69c721b57 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -105,7 +105,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
-  - [Release 2.16 (2023-05-21)](release-notes/rl-2.16.md)
+  - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)
   - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)

From ebed259768d481190ccbaa69a80ef68947ef109f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2023 17:09:38 +0200
Subject: [PATCH 056/183] Bump version

---
 .version                       | 2 +-
 maintainers/release-process.md | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.version b/.version
index 752490696..d76bd2ba3 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.16.0
+2.17.0
diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index ec9e96489..d85266b81 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -119,8 +119,7 @@ release:
   TODO: This script requires the right AWS credentials. Document.
 
   TODO: This script currently requires a
-  `/home/eelco/Dev/nix-pristine` and
-  `/home/eelco/Dev/nixpkgs-pristine`.
+  `/home/eelco/Dev/nix-pristine`.
 
   TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
 
@@ -141,7 +140,7 @@ release:
   $ git checkout master
   $ git pull
   $ NEW_VERSION=2.13.0
-  $ echo -n $NEW_VERSION > .version
+  $ echo $NEW_VERSION > .version
   $ git checkout -b bump-$NEW_VERSION
   $ git commit -a -m 'Bump version'
   $ git push --set-upstream origin bump-$NEW_VERSION

From 38ce40c66c54bfb48104e19e0e874cc047bc2092 Mon Sep 17 00:00:00 2001
From: svenz <sven.zimm@gmx.net>
Date: Thu, 1 Jun 2023 15:20:31 +0200
Subject: [PATCH 057/183] Update install-multi-user.sh

delete duplicate word 'with' in status message
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 79deb2819..c11783158 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -880,7 +880,7 @@ configure_shell_profile() {
         fi
     done
 
-    task "Setting up shell profiles for Fish with with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}"
+    task "Setting up shell profiles for Fish with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}"
     for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do
         if [ ! -d "$fish_prefix" ]; then
             # this specific prefix (ie: /etc/fish) is very likely to exist

From b292177eec7e87dcd83fd634b4d8b43de5544c5e Mon Sep 17 00:00:00 2001
From: Shamrock Lee <44064051+ShamrockLee@users.noreply.github.com>
Date: Fri, 14 May 2021 01:33:03 +0800
Subject: [PATCH 058/183] Add nix-channel --list-generations

Add support to --list-generations
as another way to say
nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations
the way we did for nix-channel --rollback [generation id]
---
 doc/manual/src/command-ref/nix-channel.md | 11 ++++++++++-
 src/nix-channel/nix-channel.cc            |  8 ++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md
index a210583ae..025f758e7 100644
--- a/doc/manual/src/command-ref/nix-channel.md
+++ b/doc/manual/src/command-ref/nix-channel.md
@@ -4,7 +4,7 @@
 
 # Synopsis
 
-`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] }
+`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] }
 
 # Description
 
@@ -39,6 +39,15 @@ This command has the following operations:
     for `nix-env` operations (by symlinking them from the directory
     `~/.nix-defexpr`).
 
+  - `--list-generations`\
+    Prints a list of all the current existing generations for the
+    channel profile.
+
+    Works the same way as
+    ```
+    nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations
+    ```
+
   - `--rollback` \[*generation*\]\
     Reverts the previous call to `nix-channel
                     --update`. Optionally, you can specify a specific channel generation
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 740737ffe..c1c8edd1d 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv)
             cRemove,
             cList,
             cUpdate,
+            cListGenerations,
             cRollback
         } cmd = cNone;
         std::vector<std::string> args;
@@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv)
                 cmd = cList;
             } else if (*arg == "--update") {
                 cmd = cUpdate;
+            } else if (*arg == "--list-generations") {
+                cmd = cListGenerations;
             } else if (*arg == "--rollback") {
                 cmd = cRollback;
             } else {
@@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv)
             case cUpdate:
                 update(StringSet(args.begin(), args.end()));
                 break;
+            case cListGenerations:
+                if (!args.empty())
+                    throw UsageError("'--list-generations' expects no arguments");
+                std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush;
+                break;
             case cRollback:
                 if (args.size() > 1)
                     throw UsageError("'--rollback' has at most one argument");

From 331f0967c4c0b4096c6c34632261a02854f407ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 2 Jun 2023 10:14:53 +0200
Subject: [PATCH 059/183] Add a release note for `nix-channel
 --list-generations`

---
 doc/manual/src/release-notes/rl-next.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 78ae99f4b..bde9057c6 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,2 +1,3 @@
 # Release X.Y (202?-??-??)
 
+- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand

From 0101ce0d9630ee62b2c96800dbcd42fba7c0bf6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 2 Jun 2023 10:20:54 +0200
Subject: [PATCH 060/183] Test `nix-channel --list-generations`

Rough test, but the feature is a fairly trivial addition on top of
`nix-profile --list-generations`, so it should be enough
---
 tests/nix-channel.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index dbb3114f1..b5d935004 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -8,6 +8,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile
 nix-channel --add http://foo/bar xyzzy
 nix-channel --list | grepQuiet http://foo/bar
 nix-channel --remove xyzzy
+[[ $(nix-channel --list-generations | wc -l) == 1 ]]
 
 [ -e $TEST_HOME/.nix-channels ]
 [ "$(cat $TEST_HOME/.nix-channels)" = '' ]
@@ -38,6 +39,7 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix
 # Test the update action.
 nix-channel --add file://$TEST_ROOT/foo
 nix-channel --update
+[[ $(nix-channel --list-generations | wc -l) == 2 ]]
 
 # Do a query.
 nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml

From 48632e6139889907999a253b36c1786cf085c43f Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 2 Jun 2023 03:04:13 -0600
Subject: [PATCH 061/183] Also set the PER_LINUX32 personality flag on
 armv5tel-linux

---
 src/libstore/build/personality.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc
index 4ad477869..1a6201758 100644
--- a/src/libstore/build/personality.cc
+++ b/src/libstore/build/personality.cc
@@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
                 && (std::string_view(SYSTEM) == "x86_64-linux"
                     || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
             || system == "armv7l-linux"
-            || system == "armv6l-linux")
+            || system == "armv6l-linux"
+            || system == "armv5tel-linux")
         {
             if (personality(PER_LINUX32) == -1)
                 throw SysError("cannot set 32-bit personality");

From a15b2c01c0bac226ed34d8ba7e10a563c4a54890 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Fri, 2 Jun 2023 15:38:17 +0000
Subject: [PATCH 062/183] Document manual migration for
 use-xdg-base-directories (#8044)

* Document manual migration for use-xdg-base-directories

As there's currently no automatic migration for use-xdg-base-directories
option, add instructions for manual migration to the option's
description.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libstore/globals.hh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 31dfe5b4e..07f524858 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -995,6 +995,18 @@ public:
           | `~/.nix-profile`  | `$XDG_STATE_HOME/nix/profile`  |
           | `~/.nix-defexpr`  | `$XDG_STATE_HOME/nix/defexpr`  |
           | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
+
+          If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option.
+          If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`.
+          This can be achieved with the following shell commands:
+
+          ```sh
+          nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix
+          mkdir -p $nix_state_home
+          mv $HOME/.nix-profile $nix_state_home/profile
+          mv $HOME/.nix-defexpr $nix_state_home/defexpr
+          mv $HOME/.nix-channels $nix_state_home/channels
+          ```
         )"
     };
 };

From b64450ed4e25ede3ce83d675c44dcf1ba524790a Mon Sep 17 00:00:00 2001
From: figsoda <figsoda@pm.me>
Date: Sun, 4 Jun 2023 14:50:47 -0400
Subject: [PATCH 063/183] Fix code block formatting in man page

---
 src/nix/nix.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/nix.md b/src/nix/nix.md
index 8a850ae83..6d9e40dbc 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -102,6 +102,7 @@ way:
   available in the flake. If this is undesirable, specify `path:<directory>` explicitly;
 
   For example, if `/foo/bar` is a git repository with the following structure:
+
   ```
   .
   └── baz

From b37dd43db4f0c9d675653c7d423407f368298173 Mon Sep 17 00:00:00 2001
From: Emily Trau <emily@downunderctf.com>
Date: Mon, 5 Jun 2023 20:18:24 -0700
Subject: [PATCH 064/183] Add missing <sys/select.h> include

`select()` may not be ambiently available for use on every platform
---
 src/nix/daemon.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index c1a91c63d..9fe9b3b1e 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -24,6 +24,7 @@
 #include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/un.h>
+#include <sys/select.h>
 #include <errno.h>
 #include <pwd.h>
 #include <grp.h>

From f5c6b299407e2ad5d05670bf4e89e18ebb113070 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Jun 2023 15:43:12 +0200
Subject: [PATCH 065/183] Fix SourcePath::resolveSymlinks()

This fixes handling of symlinks that start with '..', and symlink
targets that contain symlinks themselves.
---
 src/libfetchers/input-accessor.cc | 30 ++++++++++++++++++------------
 tests/eval.sh                     |  6 ++++++
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index f9909c218..f37a8058b 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -75,22 +75,28 @@ SourcePath SourcePath::resolveSymlinks() const
 
     int linksAllowed = 1024;
 
-    for (auto & component : path) {
-        res.path.push(component);
-        while (true) {
-            if (auto st = res.maybeLstat()) {
+    std::list<std::string> todo;
+    for (auto & c : path)
+        todo.push_back(std::string(c));
+
+    while (!todo.empty()) {
+        auto c = *todo.begin();
+        todo.pop_front();
+        if (c == "" || c == ".")
+            ;
+        else if (c == "..")
+            res.path.pop();
+        else {
+            res.path.push(c);
+            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
                 if (!linksAllowed--)
                     throw Error("infinite symlink recursion in path '%s'", path);
-                if (st->type != InputAccessor::tSymlink) break;
                 auto target = res.readLink();
+                res.path.pop();
                 if (hasPrefix(target, "/"))
-                    res = CanonPath(target);
-                else {
-                    res.path.pop();
-                    res.path.extend(CanonPath(target));
-                }
-            } else
-                break;
+                    res.path = CanonPath::root;
+                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
+            }
         }
     }
 
diff --git a/tests/eval.sh b/tests/eval.sh
index 066d8fc36..b81bb1e2c 100644
--- a/tests/eval.sh
+++ b/tests/eval.sh
@@ -35,3 +35,9 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true'
 # Check that symlink cycles don't cause a hang.
 ln -sfn cycle.nix $TEST_ROOT/cycle.nix
 (! nix eval --file $TEST_ROOT/cycle.nix)
+
+# Check that relative symlinks are resolved correctly.
+mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo
+ln -sfn ../xyzzy $TEST_ROOT/foo/bar
+printf 123 > $TEST_ROOT/xyzzy/default.nix
+[[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]]

From 42e908a3081a040619d5f65ce52d08cb1e29397d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Jun 2023 13:56:43 +0200
Subject: [PATCH 066/183] Upload fallback-paths.nix as part of the release

https://github.com/NixOS/nixpkgs/pull/235463
---
 maintainers/upload-release.pl | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 07b19c13f..2c6f8ad4f 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -179,6 +179,22 @@ if ($isLatest) {
     system("docker manifest push nixos/nix:latest") == 0 or die;
 }
 
+# Upload nix-fallback-paths.nix.
+sub getStorePath {
+    my ($jobName) = @_;
+    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+    return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
+}
+
+write_file("$tmpDir/fallback-paths.nix",
+    "{\n" .
+    "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+    "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+    "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+    "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+    "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+    "}\n");
+
 # Upload release files to S3.
 for my $fn (glob "$tmpDir/*") {
     my $name = basename($fn);
@@ -199,24 +215,6 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-# Print new nix-fallback-paths.nix.
-if ($isLatest) {
-    sub getStorePath {
-        my ($jobName) = @_;
-        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-        return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
-    }
-
-    print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" .
-               "{\n" .
-               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-               "}\n";
-}
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",

From d14f993a768452e6dfc34e3534aac8878fc711e6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Jun 2023 14:03:24 +0200
Subject: [PATCH 067/183] Fix MIME type

---
 maintainers/upload-release.pl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 2c6f8ad4f..4263a6cd2 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -205,8 +205,7 @@ for my $fn (glob "$tmpDir/*") {
         my $configuration = ();
         $configuration->{content_type} = "application/octet-stream";
 
-        if ($fn =~ /.sha256|install/) {
-            # Text files
+        if ($fn =~ /.sha256|install|\.nix$/) {
             $configuration->{content_type} = "text/plain";
         }
 

From 76e032bcf8445dc6db1ebfa1e7a8a9b377dd4242 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Jun 2023 15:37:26 +0200
Subject: [PATCH 068/183] Upload the manual to releases.nixos.org

Fixes #8451.
---
 maintainers/upload-release.pl | 39 +++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 4263a6cd2..ebc536f12 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -80,6 +80,38 @@ my $s3_us = Net::Amazon::S3->new(
 
 my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
+sub getStorePath {
+    my ($jobName, $output) = @_;
+    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+    return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'";
+}
+
+sub copyManual {
+    my $manual = getStorePath("build.x86_64-linux", "doc");
+    print "$manual\n";
+
+    my $manualNar = "$tmpDir/$releaseName-manual.nar.xz";
+    print "$manualNar\n";
+
+    unless (-e $manualNar) {
+        system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0
+            or die "unable to fetch $manual\n";
+        rename("$manualNar.tmp", $manualNar) or die;
+    }
+
+    unless (-e "$tmpDir/manual") {
+        system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0
+            or die "unable to unpack $manualNar\n";
+        rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die;
+        system("rm -rf '$tmpDir/manual.tmp'") == 0 or die;
+    }
+
+    system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
+        or die "syncing manual to S3\n";
+}
+
+copyManual;
+
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
 
@@ -180,12 +212,6 @@ if ($isLatest) {
 }
 
 # Upload nix-fallback-paths.nix.
-sub getStorePath {
-    my ($jobName) = @_;
-    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
-}
-
 write_file("$tmpDir/fallback-paths.nix",
     "{\n" .
     "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
@@ -198,6 +224,7 @@ write_file("$tmpDir/fallback-paths.nix",
 # Upload release files to S3.
 for my $fn (glob "$tmpDir/*") {
     my $name = basename($fn);
+    next if $name eq "manual";
     my $dstKey = "$releaseDir/" . $name;
     unless (defined $releasesBucket->head_key($dstKey)) {
         print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";

From db680e0e5751828e788bff4becd89886bde71480 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 8 Jun 2023 02:00:05 +0200
Subject: [PATCH 069/183] refine wording on the purpose of the Nix language

packages and configurations are not really a concept in Nix or the Nix language. the idea of transforming files into other files clearly captures what it's all about, and the new phrasing should make the term "derivation" more obvious both in terms of meaning and origin.
---
 doc/manual/src/language/index.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 3eabe1a02..69ebc70c0 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -4,9 +4,7 @@ The Nix language is
 
 - *domain-specific*
 
-  It only exists for the Nix package manager:
-  to describe packages and configurations as well as their variants and compositions.
-  It is not intended for general purpose use.
+  Its purpose is to conveniently create and compose precise descriptions of how contents of existing files are used to derive new files.
 
 - *declarative*
 
@@ -25,7 +23,7 @@ The Nix language is
 
 - *lazy*
 
-  Expressions are only evaluated when their value is needed.
+  Values are only computed when they are needed.
 
 - *dynamically typed*
 

From 3c78920f7358957dba37a379e9d0b062dd3192e2 Mon Sep 17 00:00:00 2001
From: Andrea Bedini <andrea@andreabedini.com>
Date: Fri, 9 Jun 2023 17:53:18 +0800
Subject: [PATCH 070/183] Parse TOML timestamps (#8120)

Currently `fromTOML` throws an exception when encountering a timestamp
since the Nix language lacks a way to represent them.

This patch changes this beaviour and makes `fromTOML` parse timestamps as
attrsets of the format

  { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }

This is guarded by an experimental feature flag to leave room for iterating on the representation.
---
 src/libexpr/primops/fromTOML.cc               |  16 ++-
 src/libutil/experimental-features.cc          |  11 +-
 src/libutil/experimental-features.hh          |   1 +
 tests/lang/eval-fail-fromTOML-timestamps.nix  | 130 ++++++++++++++++++
 tests/lang/eval-okay-fromTOML-timestamps.exp  |   1 +
 .../lang/eval-okay-fromTOML-timestamps.flags  |   1 +
 tests/lang/eval-okay-fromTOML-timestamps.nix  | 130 ++++++++++++++++++
 7 files changed, 286 insertions(+), 4 deletions(-)
 create mode 100644 tests/lang/eval-fail-fromTOML-timestamps.nix
 create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.exp
 create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.flags
 create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.nix

diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 8a5231781..e2a8b3c3a 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -3,6 +3,8 @@
 
 #include "../../toml11/toml.hpp"
 
+#include <sstream>
+
 namespace nix {
 
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)
@@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
             case toml::value_t::offset_datetime:
             case toml::value_t::local_date:
             case toml::value_t::local_time:
-                // We fail since Nix doesn't have date and time types
-                throw std::runtime_error("Dates and times are not supported");
+                {
+                    if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
+                        auto attrs = state.buildBindings(2);
+                        attrs.alloc("_type").mkString("timestamp");
+                        std::ostringstream s;
+                        s << t;
+                        attrs.alloc("value").mkString(s.str());
+                        v.mkAttrs(attrs);
+                    } else {
+                        throw std::runtime_error("Dates and times are not supported");
+                    }
+                }
                 break;;
             case toml::value_t::empty:
                 v.mkNull();
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 4bf545d7b..c4642d333 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -214,6 +214,13 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
                 derivations that are themselves derivations outputs.
         )",
     },
+    {
+        .tag = Xp::ParseTomlTimestamps,
+        .name = "parse-toml-timestamps",
+        .description = R"(
+            Allow parsing of timestamps in builtins.fromTOML.
+        )",
+    },
 }};
 
 static_assert(
@@ -248,7 +255,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag)
     return xpFeatureDetails[(size_t)tag].name;
 }
 
-nlohmann::json documentExperimentalFeatures() 
+nlohmann::json documentExperimentalFeatures()
 {
     StringMap res;
     for (auto & xpFeature : xpFeatureDetails)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 409100592..892c6c371 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -30,6 +30,7 @@ enum struct ExperimentalFeature
     DiscardReferences,
     DaemonTrustOverride,
     DynamicDerivations,
+    ParseTomlTimestamps,
 };
 
 /**
diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/lang/eval-fail-fromTOML-timestamps.nix
new file mode 100644
index 000000000..74cff9470
--- /dev/null
+++ b/tests/lang/eval-fail-fromTOML-timestamps.nix
@@ -0,0 +1,130 @@
+builtins.fromTOML ''
+  key = "value"
+  bare_key = "value"
+  bare-key = "value"
+  1234 = "value"
+
+  "127.0.0.1" = "value"
+  "character encoding" = "value"
+  "ʎǝʞ" = "value"
+  'key2' = "value"
+  'quoted "value"' = "value"
+
+  name = "Orange"
+
+  physical.color = "orange"
+  physical.shape = "round"
+  site."google.com" = true
+
+  # This is legal according to the spec, but cpptoml doesn't handle it.
+  #a.b.c = 1
+  #a.d = 2
+
+  str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+  int1 = +99
+  int2 = 42
+  int3 = 0
+  int4 = -17
+  int5 = 1_000
+  int6 = 5_349_221
+  int7 = 1_2_3_4_5
+
+  hex1 = 0xDEADBEEF
+  hex2 = 0xdeadbeef
+  hex3 = 0xdead_beef
+
+  oct1 = 0o01234567
+  oct2 = 0o755
+
+  bin1 = 0b11010110
+
+  flt1 = +1.0
+  flt2 = 3.1415
+  flt3 = -0.01
+  flt4 = 5e+22
+  flt5 = 1e6
+  flt6 = -2E-2
+  flt7 = 6.626e-34
+  flt8 = 9_224_617.445_991_228_313
+
+  bool1 = true
+  bool2 = false
+
+  odt1 = 1979-05-27T07:32:00Z
+  odt2 = 1979-05-27T00:32:00-07:00
+  odt3 = 1979-05-27T00:32:00.999999-07:00
+  odt4 = 1979-05-27 07:32:00Z
+  ldt1 = 1979-05-27T07:32:00
+  ldt2 = 1979-05-27T00:32:00.999999
+  ld1 = 1979-05-27
+  lt1 = 07:32:00
+  lt2 = 00:32:00.999999
+
+  arr1 = [ 1, 2, 3 ]
+  arr2 = [ "red", "yellow", "green" ]
+  arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+  arr4 = [ "all", 'strings', """are the same""", ''''type'''']
+  arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+
+  arr7 = [
+    1, 2, 3
+  ]
+
+  arr8 = [
+    1,
+    2, # this is ok
+  ]
+
+  [table-1]
+  key1 = "some string"
+  key2 = 123
+
+
+  [table-2]
+  key1 = "another string"
+  key2 = 456
+
+  [dog."tater.man"]
+  type.name = "pug"
+
+  [a.b.c]
+  [ d.e.f ]
+  [ g .  h  . i ]
+  [ j . "ʞ" . 'l' ]
+  [x.y.z.w]
+
+  name = { first = "Tom", last = "Preston-Werner" }
+  point = { x = 1, y = 2 }
+  animal = { type.name = "pug" }
+
+  [[products]]
+  name = "Hammer"
+  sku = 738594937
+
+  [[products]]
+
+  [[products]]
+  name = "Nail"
+  sku = 284758393
+  color = "gray"
+
+  [[fruit]]
+    name = "apple"
+
+    [fruit.physical]
+      color = "red"
+      shape = "round"
+
+    [[fruit.variety]]
+      name = "red delicious"
+
+    [[fruit.variety]]
+      name = "granny smith"
+
+  [[fruit]]
+    name = "banana"
+
+    [[fruit.variety]]
+      name = "plantain"
+''
diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/lang/eval-okay-fromTOML-timestamps.exp
new file mode 100644
index 000000000..08b3c69a6
--- /dev/null
+++ b/tests/lang/eval-okay-fromTOML-timestamps.exp
@@ -0,0 +1 @@
+{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; }
diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/lang/eval-okay-fromTOML-timestamps.flags
new file mode 100644
index 000000000..9ed39dc6b
--- /dev/null
+++ b/tests/lang/eval-okay-fromTOML-timestamps.flags
@@ -0,0 +1 @@
+--extra-experimental-features parse-toml-timestamps
diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/lang/eval-okay-fromTOML-timestamps.nix
new file mode 100644
index 000000000..74cff9470
--- /dev/null
+++ b/tests/lang/eval-okay-fromTOML-timestamps.nix
@@ -0,0 +1,130 @@
+builtins.fromTOML ''
+  key = "value"
+  bare_key = "value"
+  bare-key = "value"
+  1234 = "value"
+
+  "127.0.0.1" = "value"
+  "character encoding" = "value"
+  "ʎǝʞ" = "value"
+  'key2' = "value"
+  'quoted "value"' = "value"
+
+  name = "Orange"
+
+  physical.color = "orange"
+  physical.shape = "round"
+  site."google.com" = true
+
+  # This is legal according to the spec, but cpptoml doesn't handle it.
+  #a.b.c = 1
+  #a.d = 2
+
+  str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+  int1 = +99
+  int2 = 42
+  int3 = 0
+  int4 = -17
+  int5 = 1_000
+  int6 = 5_349_221
+  int7 = 1_2_3_4_5
+
+  hex1 = 0xDEADBEEF
+  hex2 = 0xdeadbeef
+  hex3 = 0xdead_beef
+
+  oct1 = 0o01234567
+  oct2 = 0o755
+
+  bin1 = 0b11010110
+
+  flt1 = +1.0
+  flt2 = 3.1415
+  flt3 = -0.01
+  flt4 = 5e+22
+  flt5 = 1e6
+  flt6 = -2E-2
+  flt7 = 6.626e-34
+  flt8 = 9_224_617.445_991_228_313
+
+  bool1 = true
+  bool2 = false
+
+  odt1 = 1979-05-27T07:32:00Z
+  odt2 = 1979-05-27T00:32:00-07:00
+  odt3 = 1979-05-27T00:32:00.999999-07:00
+  odt4 = 1979-05-27 07:32:00Z
+  ldt1 = 1979-05-27T07:32:00
+  ldt2 = 1979-05-27T00:32:00.999999
+  ld1 = 1979-05-27
+  lt1 = 07:32:00
+  lt2 = 00:32:00.999999
+
+  arr1 = [ 1, 2, 3 ]
+  arr2 = [ "red", "yellow", "green" ]
+  arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+  arr4 = [ "all", 'strings', """are the same""", ''''type'''']
+  arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+
+  arr7 = [
+    1, 2, 3
+  ]
+
+  arr8 = [
+    1,
+    2, # this is ok
+  ]
+
+  [table-1]
+  key1 = "some string"
+  key2 = 123
+
+
+  [table-2]
+  key1 = "another string"
+  key2 = 456
+
+  [dog."tater.man"]
+  type.name = "pug"
+
+  [a.b.c]
+  [ d.e.f ]
+  [ g .  h  . i ]
+  [ j . "ʞ" . 'l' ]
+  [x.y.z.w]
+
+  name = { first = "Tom", last = "Preston-Werner" }
+  point = { x = 1, y = 2 }
+  animal = { type.name = "pug" }
+
+  [[products]]
+  name = "Hammer"
+  sku = 738594937
+
+  [[products]]
+
+  [[products]]
+  name = "Nail"
+  sku = 284758393
+  color = "gray"
+
+  [[fruit]]
+    name = "apple"
+
+    [fruit.physical]
+      color = "red"
+      shape = "round"
+
+    [[fruit.variety]]
+      name = "red delicious"
+
+    [[fruit.variety]]
+      name = "granny smith"
+
+  [[fruit]]
+    name = "banana"
+
+    [[fruit.variety]]
+      name = "plantain"
+''

From e54538c461e993827d9fbe3b8883d3887f184798 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jun 2023 16:09:29 +0200
Subject: [PATCH 071/183] restoreMountNamespace(): Restore the original root
 directory

This is necessary when we're in a chroot environment, where the
process root is not the same as the root of the mount namespace
(e.g. in nixos-enter).

Fixes #7602.
---
 src/libutil/util.cc | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index aa0a154fd..26f9dc8a8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1853,6 +1853,7 @@ void setStackSize(size_t stackSize)
 
 #if __linux__
 static AutoCloseFD fdSavedMountNamespace;
+static AutoCloseFD fdSavedRoot;
 #endif
 
 void saveMountNamespace()
@@ -1860,10 +1861,11 @@ void saveMountNamespace()
 #if __linux__
     static std::once_flag done;
     std::call_once(done, []() {
-        AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY);
-        if (!fd)
+        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
+        if (!fdSavedMountNamespace)
             throw SysError("saving parent mount namespace");
-        fdSavedMountNamespace = std::move(fd);
+
+        fdSavedRoot = open("/proc/self/root", O_RDONLY);
     });
 #endif
 }
@@ -1876,9 +1878,16 @@ void restoreMountNamespace()
 
         if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
             throw SysError("restoring parent mount namespace");
-        if (chdir(savedCwd.c_str()) == -1) {
-            throw SysError("restoring cwd");
+
+        if (fdSavedRoot) {
+            if (fchdir(fdSavedRoot.get()))
+                throw SysError("chdir into saved root");
+            if (chroot("."))
+                throw SysError("chroot into saved root");
         }
+
+        if (chdir(savedCwd.c_str()) == -1)
+            throw SysError("restoring cwd");
     } catch (Error & e) {
         debug(e.msg());
     }

From c51f3f1eb250dc006af3de340b1d20b759fed9d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 10 Jun 2023 13:55:58 +0200
Subject: [PATCH 072/183] nix actually needs c++20 now

---
 doc/manual/src/installation/prerequisites-source.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index 5a708f11b..d4babf1ea 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -10,7 +10,7 @@
   - Bash Shell. The `./configure` script relies on bashisms, so Bash is
     required.
 
-  - A version of GCC or Clang that supports C++17.
+  - A version of GCC or Clang that supports C++20.
 
   - `pkg-config` to locate dependencies. If your distribution does not
     provide it, you can get it from

From 08089fdd3284fcb27bcec347d6bf88ec2a79a8a0 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Sun, 11 Jun 2023 13:33:38 -0400
Subject: [PATCH 073/183] fix: Do not apply default installables when using
 --stdin

---
 src/libcmd/installables.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index a2b882355..684d76a57 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -730,9 +730,9 @@ void RawInstallablesCommand::run(ref<Store> store)
         while (std::cin >> word) {
             rawInstallables.emplace_back(std::move(word));
         }
+    } else {
+        applyDefaultInstallables(rawInstallables);
     }
-
-    applyDefaultInstallables(rawInstallables);
     run(store, std::move(rawInstallables));
 }
 

From 8ec1ba02109eee7e8dfd89f3fbbf060497940469 Mon Sep 17 00:00:00 2001
From: Michal Sojka <michal.sojka@cvut.cz>
Date: Sun, 11 Jun 2023 16:57:50 +0200
Subject: [PATCH 074/183] Register all PrimOps via the Info structure

This will allow documenting them (in later commits).

Note that we keep the old constructor even if it is no longer used by
Nix code, because it is used in tests/plugins/plugintest.cc, which
suggests that it might be used by some external plugin.
---
 src/libexpr/primops/context.cc        | 30 ++++++++++++++++++++++-----
 src/libexpr/primops/fetchMercurial.cc |  6 +++++-
 src/libexpr/primops/fetchTree.cc      |  6 +++++-
 src/libexpr/primops/fromTOML.cc       |  6 +++++-
 4 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 07bf400cf..8ae68e570 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
     v.mkString(*s);
 }
 
-static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+static RegisterPrimOp primop_unsafeDiscardStringContext({
+    .name = "__unsafeDiscardStringContext",
+    .arity = 1,
+    .fun = prim_unsafeDiscardStringContext
+});
 
 
 static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -22,7 +26,11 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args,
     v.mkBool(!context.empty());
 }
 
-static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
+static RegisterPrimOp primop_hasContext({
+    .name = "__hasContext",
+    .arity = 1,
+    .fun = prim_hasContext
+});
 
 
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
@@ -51,7 +59,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
     v.mkString(*s, context2);
 }
 
-static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+static RegisterPrimOp primop_unsafeDiscardOutputDependency({
+    .name = "__unsafeDiscardOutputDependency",
+    .arity = 1,
+    .fun = prim_unsafeDiscardOutputDependency
+});
 
 
 /* Extract the context of a string as a structured Nix value.
@@ -119,7 +131,11 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
     v.mkAttrs(attrs);
 }
 
-static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
+static RegisterPrimOp primop_getContext({
+    .name = "__getContext",
+    .arity = 1,
+    .fun = prim_getContext
+});
 
 
 /* Append the given context to a given string.
@@ -192,6 +208,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
     v.mkString(orig, context);
 }
 
-static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);
+static RegisterPrimOp primop_appendContext({
+    .name = "__appendContext",
+    .arity = 2,
+    .fun = prim_appendContext
+});
 
 }
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 2c0d98e74..322692b52 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     state.allowPath(tree.storePath);
 }
 
-static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);
+static RegisterPrimOp r_fetchMercurial({
+    .name = "fetchMercurial",
+    .arity = 1,
+    .fun = prim_fetchMercurial
+});
 
 }
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index fe880aaa8..be8159cc8 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -194,7 +194,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
 }
 
 // FIXME: document
-static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
+static RegisterPrimOp primop_fetchTree({
+    .name = "fetchTree",
+    .arity = 1,
+    .fun = prim_fetchTree
+});
 
 static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
     const std::string & who, bool unpack, std::string name)
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index e2a8b3c3a..4e6fcdffa 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -90,6 +90,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
     }
 }
 
-static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML);
+static RegisterPrimOp primop_fromTOML({
+    .name = "fromTOML",
+    .arity = 1,
+    .fun = prim_fromTOML
+});
 
 }

From 87c66f6b0f35bc59749cad6bfb3dc2e23de8c86b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2023 11:10:55 +0200
Subject: [PATCH 075/183] Don't include uds-remote-store.md from a header file

Closes #8484.
---
 src/libstore/uds-remote-store.cc | 8 ++++++++
 src/libstore/uds-remote-store.hh | 7 +------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 0fb7c38e9..69dae2da5 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -13,6 +13,14 @@
 
 namespace nix {
 
+std::string UDSRemoteStoreConfig::doc()
+{
+    return
+        #include "uds-remote-store.md"
+        ;
+}
+
+
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
     , LocalFSStoreConfig(params)
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index bd1dcb67c..3bbab371c 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
 
     const std::string name() override { return "Local Daemon Store"; }
 
-    std::string doc() override
-    {
-        return
-          #include "uds-remote-store.md"
-          ;
-    }
+    std::string doc() override;
 };
 
 class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

From 0e3849dc650250e36fb69c197a8a1fffe4f7d480 Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tomberek@gmail.com>
Date: Mon, 12 Jun 2023 08:40:17 -0400
Subject: [PATCH 076/183] test: add test for non-defaulting for stding
 installable input

---
 src/libcmd/installables.cc | 2 +-
 tests/build.sh             | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 684d76a57..10b077fb5 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -701,7 +701,7 @@ RawInstallablesCommand::RawInstallablesCommand()
 {
     addFlag({
         .longName = "stdin",
-        .description = "Read installables from the standard input.",
+        .description = "Read installables from the standard input. No default installable applied.",
         .handler = {&readFromStdIn, true}
     });
 
diff --git a/tests/build.sh b/tests/build.sh
index 697aff0f9..8ae20f0df 100644
--- a/tests/build.sh
+++ b/tests/build.sh
@@ -129,3 +129,7 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status
     (.drvPath | match(".*multiple-outputs-e.drv")) and
     (.outputs | keys == ["a_a", "b"]))
 '
+
+# Make sure that `--stdin` works and does not apply any defaults
+printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []'
+printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")'

From f961b04484a5770d12ad7866d86d1d1f8e7c687c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 22:56:41 +0000
Subject: [PATCH 077/183] Bump zeebe-io/backport-action from 1.3.0 to 1.3.1

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.3.0 to 1.3.1.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v1.3.0...v1.3.1)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 37966bab2..816474ed5 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.3.0
+        uses: zeebe-io/backport-action@v1.3.1
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From 3402b650cdce318e42acd4dc34f42423cafef587 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jun 2023 15:06:12 +0200
Subject: [PATCH 078/183] Add a generic check for rev attribute mismatches

---
 src/libfetchers/fetchers.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 91db3a9eb..2860c1ceb 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -159,6 +159,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
                 input.to_string(), *prevLastModified);
     }
 
+    if (auto prevRev = getRev()) {
+        if (input.getRev() != prevRev)
+            throw Error("'rev' attribute mismatch in input '%s', expected %s",
+                input.to_string(), prevRev->gitRev());
+    }
+
     if (auto prevRevCount = getRevCount()) {
         if (input.getRevCount() != prevRevCount)
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",

From 1ad3328c5efea041990fa82e6ad24ae2b4e81c24 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jun 2023 14:26:30 +0200
Subject: [PATCH 079/183] Allow tarball URLs to redirect to a lockable
 immutable URL

Previously, for tarball flakes, we recorded the original URL of the
tarball flake, rather than the URL to which it ultimately
redirects. Thus, a flake URL like
http://example.org/patchelf-latest.tar that redirects to
http://example.org/patchelf-<revision>.tar was not really usable. We
couldn't record the redirected URL, because sites like GitHub redirect
to CDN URLs that we can't rely on to be stable.

So now we use the redirected URL only if the server returns the
`x-nix-is-immutable` or `x-amz-meta-nix-is-immutable` headers in its
response.
---
 flake.nix                        |  2 +
 src/libcmd/common-eval-args.cc   |  2 +-
 src/libexpr/parser.y             |  2 +-
 src/libexpr/primops/fetchTree.cc |  2 +-
 src/libfetchers/attrs.hh         |  1 +
 src/libfetchers/fetchers.hh      | 10 +++-
 src/libfetchers/github.cc        | 10 ++--
 src/libfetchers/tarball.cc       | 68 +++++++++++++++++++-------
 src/libstore/filetransfer.cc     | 22 +++++++--
 src/libstore/filetransfer.hh     |  4 ++
 tests/nixos/tarball-flakes.nix   | 84 ++++++++++++++++++++++++++++++++
 11 files changed, 177 insertions(+), 30 deletions(-)
 create mode 100644 tests/nixos/tarball-flakes.nix

diff --git a/flake.nix b/flake.nix
index a4ee80b32..bdbf54169 100644
--- a/flake.nix
+++ b/flake.nix
@@ -590,6 +590,8 @@
 
         tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix;
 
+        tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix;
+
         tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix;
 
         tests.setuid = lib.genAttrs
diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index ff3abd534..7f97364a1 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -165,7 +165,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto storePath = fetchers::downloadTarball(
-            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath;
+            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath;
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 4d981712a..3b545fd84 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -793,7 +793,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (EvalSettings::isPseudoUrl(elem.second)) {
         try {
             auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath;
+                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath;
             res = { true, store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index fe880aaa8..b34a46fa0 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -262,7 +262,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     if (expectedHash) {
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 1a14bb023..9f885a793 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "hash.hh"
 
 #include <variant>
 
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 498ad7e4d..d0738f619 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -158,6 +158,7 @@ struct DownloadFileResult
     StorePath storePath;
     std::string etag;
     std::string effectiveUrl;
+    std::optional<std::string> immutableUrl;
 };
 
 DownloadFileResult downloadFile(
@@ -167,7 +168,14 @@ DownloadFileResult downloadFile(
     bool locked,
     const Headers & headers = {});
 
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballResult
+{
+    Tree tree;
+    time_t lastModified;
+    std::optional<std::string> immutableUrl;
+};
+
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 6c1d573ce..80598e7f8 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,21 +207,21 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
+        auto result = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         getCache()->add(
             store,
             lockedAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(lastModified)}
+                {"lastModified", uint64_t(result.lastModified)}
             },
-            tree.storePath,
+            result.tree.storePath,
             true);
 
-        return {std::move(tree.storePath), input};
+        return {result.tree.storePath, input};
     }
 };
 
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 96fe5faca..e42aca6db 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -32,7 +32,8 @@ DownloadFileResult downloadFile(
         return {
             .storePath = std::move(cached->storePath),
             .etag = getStrAttr(cached->infoAttrs, "etag"),
-            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
+            .effectiveUrl = getStrAttr(cached->infoAttrs, "url"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
     };
 
@@ -55,12 +56,14 @@ DownloadFileResult downloadFile(
     }
 
     // FIXME: write to temporary file.
-
     Attrs infoAttrs({
         {"etag", res.etag},
         {"url", res.effectiveUri},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     std::optional<StorePath> storePath;
 
     if (res.cached) {
@@ -111,10 +114,11 @@ DownloadFileResult downloadFile(
         .storePath = std::move(*storePath),
         .etag = res.etag,
         .effectiveUrl = res.effectiveUri,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
-std::pair<Tree, time_t> downloadTarball(
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -131,8 +135,9 @@ std::pair<Tree, time_t> downloadTarball(
 
     if (cached && !cached->expired)
         return {
-            Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
-            getIntAttr(cached->infoAttrs, "lastModified")
+            .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
+            .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
 
     auto res = downloadFile(store, url, name, locked, headers);
@@ -160,6 +165,9 @@ std::pair<Tree, time_t> downloadTarball(
         {"etag", res.etag},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     getCache()->add(
         store,
         inAttrs,
@@ -168,8 +176,9 @@ std::pair<Tree, time_t> downloadTarball(
         locked);
 
     return {
-        Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
-        lastModified,
+        .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
+        .lastModified = lastModified,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
@@ -189,21 +198,33 @@ struct CurlInputScheme : InputScheme
 
     virtual bool isValidURL(const ParsedURL & url) const = 0;
 
-    std::optional<Input> inputFromURL(const ParsedURL & url) const override
+    std::optional<Input> inputFromURL(const ParsedURL & _url) const override
     {
-        if (!isValidURL(url))
+        if (!isValidURL(_url))
             return std::nullopt;
 
         Input input;
 
-        auto urlWithoutApplicationScheme = url;
-        urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport;
+        auto url = _url;
+
+        url.scheme = parseUrlScheme(url.scheme).transport;
 
-        input.attrs.insert_or_assign("type", inputType());
-        input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string());
         auto narHash = url.query.find("narHash");
         if (narHash != url.query.end())
             input.attrs.insert_or_assign("narHash", narHash->second);
+
+        if (auto i = get(url.query, "rev"))
+            input.attrs.insert_or_assign("rev", *i);
+
+        if (auto i = get(url.query, "revCount"))
+            if (auto n = string2Int<uint64_t>(*i))
+                input.attrs.insert_or_assign("revCount", *n);
+
+        url.query.erase("rev");
+        url.query.erase("revCount");
+
+        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("url", url.to_string());
         return input;
     }
 
@@ -212,7 +233,8 @@ struct CurlInputScheme : InputScheme
         auto type = maybeGetStrAttr(attrs, "type");
         if (type != inputType()) return {};
 
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"};
+        // FIXME: some of these only apply to TarballInputScheme.
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"};
         for (auto & [name, value] : attrs)
             if (!allowedNames.count(name))
                 throw Error("unsupported %s input attribute '%s'", *type, name);
@@ -275,10 +297,22 @@ struct TarballInputScheme : CurlInputScheme
                     : hasTarballExtension(url.path));
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
-        return {std::move(tree.storePath), input};
+        Input input(_input);
+        auto url = getStrAttr(input.attrs, "url");
+        auto result = downloadTarball(store, url, input.getName(), false);
+
+        if (result.immutableUrl) {
+            auto immutableInput = Input::fromURL(*result.immutableUrl);
+            // FIXME: would be nice to support arbitrary flakerefs
+            // here, e.g. git flakes.
+            if (immutableInput.getType() != "tarball")
+                throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported");
+            input = immutableInput;
+        }
+
+        return {result.tree.storePath, std::move(input)};
     }
 };
 
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 2346accbe..38b691279 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
+
             static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
-            std::smatch match;
-            if (std::regex_match(line, match, statusLine)) {
+            if (std::smatch match; std::regex_match(line, match, statusLine)) {
                 result.etag = "";
                 result.data.clear();
                 result.bodySize = 0;
@@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer
                 acceptRanges = false;
                 encoding = "";
             } else {
+
                 auto i = line.find(':');
                 if (i != std::string::npos) {
                     std::string name = toLower(trim(line.substr(0, i)));
+
                     if (name == "etag") {
                         result.etag = trim(line.substr(i + 1));
                         /* Hack to work around a GitHub bug: it sends
@@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer
                             debug("shutting down on 200 HTTP response with expected ETag");
                             return 0;
                         }
-                    } else if (name == "content-encoding")
+                    }
+
+                    else if (name == "content-encoding")
                         encoding = trim(line.substr(i + 1));
+
                     else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
                         acceptRanges = true;
+
+                    else if (name == "link" || name == "x-amz-meta-link") {
+                        auto value = trim(line.substr(i + 1));
+                        static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase);
+                        if (std::smatch match; std::regex_match(value, match, linkRegex))
+                            result.immutableUrl = match.str(1);
+                        else
+                            debug("got invalid link header '%s'", value);
+                    }
                 }
             }
             return realSize;
@@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer
         {
             auto httpStatus = getHTTPStatus();
 
-            char * effectiveUriCStr;
+            char * effectiveUriCStr = nullptr;
             curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
             if (effectiveUriCStr)
                 result.effectiveUri = effectiveUriCStr;
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 378c6ff78..a3b0dde1f 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -80,6 +80,10 @@ struct FileTransferResult
     std::string effectiveUri;
     std::string data;
     uint64_t bodySize = 0;
+    /* An "immutable" URL for this resource (i.e. one whose contents
+       will never change), as returned by the `Link: <url>;
+       rel="immutable"` header. */
+    std::optional<std::string> immutableUrl;
 };
 
 class Store;
diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix
new file mode 100644
index 000000000..1d43a5d04
--- /dev/null
+++ b/tests/nixos/tarball-flakes.nix
@@ -0,0 +1,84 @@
+{ lib, config, nixpkgs, ... }:
+
+let
+  pkgs = config.nodes.machine.nixpkgs.pkgs;
+
+  root = pkgs.runCommand "nixpkgs-flake" {}
+    ''
+      mkdir -p $out/stable
+
+      set -x
+      dir=nixpkgs-${nixpkgs.shortRev}
+      cp -prd ${nixpkgs} $dir
+      # Set the correct timestamp in the tarball.
+      find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
+      tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference
+
+      echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess
+
+      echo 'Header set Link "<http://localhost/stable/${nixpkgs.rev}.tar.gz?rev=${nixpkgs.rev}&revCount=1234>; rel=\"immutable\""' > $out/stable/.htaccess
+    '';
+in
+
+{
+  name = "tarball-flakes";
+
+  nodes =
+    {
+      machine =
+        { config, pkgs, ... }:
+        { networking.firewall.allowedTCPPorts = [ 80 ];
+
+          services.httpd.enable = true;
+          services.httpd.adminAddr = "foo@example.org";
+          services.httpd.extraConfig = ''
+            ErrorLog syslog:local6
+          '';
+          services.httpd.virtualHosts."localhost" =
+            { servedDirs =
+                [ { urlPath = "/";
+                    dir = root;
+                  }
+                ];
+            };
+
+          virtualisation.writableStore = true;
+          virtualisation.diskSize = 2048;
+          virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ];
+          virtualisation.memorySize = 4096;
+          nix.settings.substituters = lib.mkForce [ ];
+          nix.extraOptions = "experimental-features = nix-command flakes";
+        };
+    };
+
+  testScript = { nodes }: ''
+    # fmt: off
+    import json
+
+    start_all()
+
+    machine.wait_for_unit("httpd.service")
+
+    out = machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz")
+    print(out)
+    info = json.loads(out)
+
+    # Check that we got redirected to the immutable URL.
+    assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz"
+
+    # Check that we got the rev and revCount attributes.
+    assert info["revision"] == "${nixpkgs.rev}"
+    assert info["revCount"] == 1234
+
+    # Check that fetching with rev/revCount/narHash succeeds.
+    machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?rev=" + info["revision"])
+    machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?revCount=" + str(info["revCount"]))
+    machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?narHash=" + info["locked"]["narHash"])
+
+    # Check that fetching fails if we provide incorrect attributes.
+    machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0")
+    machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?revCount=789")
+    machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=")
+  '';
+
+}

From c6d7c4f9ecb919cced435cf179d8ece5d25b5f06 Mon Sep 17 00:00:00 2001
From: Michal Sojka <michal.sojka@cvut.cz>
Date: Sun, 11 Jun 2023 21:38:18 +0200
Subject: [PATCH 080/183] Document fromTOML, hasContext and getContext builtins

Until now, these functions were completely missing in the Nix manual.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libexpr/primops/context.cc  | 28 ++++++++++++++++++++++++++--
 src/libexpr/primops/fromTOML.cc | 16 +++++++++++++++-
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 8ae68e570..8b3468009 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -28,7 +28,12 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args,
 
 static RegisterPrimOp primop_hasContext({
     .name = "__hasContext",
-    .arity = 1,
+    .args = {"s"},
+    .doc = R"(
+      Return `true` if string *s* has a non-empty context. The
+      context can be obtained with
+      [`getContext`](#builtins-getContext).
+    )",
     .fun = prim_hasContext
 });
 
@@ -133,7 +138,26 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
 
 static RegisterPrimOp primop_getContext({
     .name = "__getContext",
-    .arity = 1,
+    .args = {"s"},
+    .doc = R"(
+      Return the string context of *s*.
+
+      The string context tracks references to derivations within a string.
+      It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names.
+
+      Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context.
+      For example,
+
+      ```nix
+      builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}"
+      ```
+
+      evaluates to
+
+      ```
+      { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; }
+      ```
+    )",
     .fun = prim_getContext
 });
 
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 4e6fcdffa..2f4d4022e 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -92,7 +92,21 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
 
 static RegisterPrimOp primop_fromTOML({
     .name = "fromTOML",
-    .arity = 1,
+    .args = {"e"},
+    .doc = R"(
+      Convert a TOML string to a Nix value. For example,
+
+      ```nix
+      builtins.fromTOML ''
+        x=1
+        s="a"
+        [table]
+        y=2
+      ''
+      ```
+
+      returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`.
+    )",
     .fun = prim_fromTOML
 });
 

From bfb5e0bdcddc3ba4866d30c54de49248ca763951 Mon Sep 17 00:00:00 2001
From: scarf <greenscarf005@gmail.com>
Date: Wed, 14 Jun 2023 19:06:04 +0900
Subject: [PATCH 081/183] build: show UID and GID in welcome message

---
 scripts/install-multi-user.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index c11783158..2e8f2b8c3 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -700,6 +700,10 @@ EOF
 }
 
 welcome_to_nix() {
+    local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
+    local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}")
+    local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}")
+
     ok "Welcome to the Multi-User Nix Installation"
 
     cat <<EOF
@@ -713,8 +717,8 @@ manager. This will happen in a few stages:
 2. Show you what I am going to install and where. Then I will ask
    if you are ready to continue.
 
-3. Create the system users and groups that the Nix daemon uses to run
-   builds.
+3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT}
+   that the Nix daemon uses to run builds.
 
 4. Perform the basic installation of the Nix files daemon.
 

From c73daea61eba8679a3281ac8e1af1e8eeefe9452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 24 Apr 2020 17:07:29 +0200
Subject: [PATCH 082/183] darwin installer: remove the file before installing
 new one

Otherwise results into:

cp: /Library/LaunchDaemons/org.nixos.nix-daemon.plist and /nix/var/nix/profiles/default/Library/LaunchDaemons/org.nixos.nix-daemon.plist are identical (not copied).
---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 5111a5dde..0326d3415 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -100,7 +100,7 @@ poly_extra_try_me_commands() {
 poly_configure_nix_daemon_service() {
     task "Setting up the nix-daemon LaunchDaemon"
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST"
+          /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist

From 468add5aa0b5b0a686cd07aa5417817bb8799602 Mon Sep 17 00:00:00 2001
From: Daniel Asaturov <122093031+salpelter@users.noreply.github.com>
Date: Wed, 14 Jun 2023 22:09:11 +0400
Subject: [PATCH 083/183] Remove dead code (#8504)

`filesystem.cc` is the only place where `createSymlink()` is used with three arguments:
in the definition of `replaceSymlink()` with three parameters that _is not used at all_.

Closes #8495
---
 src/libutil/filesystem.cc | 17 +++--------------
 src/libutil/util.hh       |  6 ++----
 2 files changed, 5 insertions(+), 18 deletions(-)

diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc
index 56be76ecc..11cc0c0e7 100644
--- a/src/libutil/filesystem.cc
+++ b/src/libutil/filesystem.cc
@@ -63,30 +63,19 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
     return {std::move(fd), tmpl};
 }
 
-void createSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
+void createSymlink(const Path & target, const Path & link)
 {
     if (symlink(target.c_str(), link.c_str()))
         throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-    if (mtime) {
-        struct timeval times[2];
-        times[0].tv_sec = *mtime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = *mtime;
-        times[1].tv_usec = 0;
-        if (lutimes(link.c_str(), times))
-            throw SysError("setting time of symlink '%s'", link);
-    }
 }
 
-void replaceSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
+void replaceSymlink(const Path & target, const Path & link)
 {
     for (unsigned int n = 0; true; n++) {
         Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
 
         try {
-            createSymlink(target, tmp, mtime);
+            createSymlink(target, tmp);
         } catch (SysError & e) {
             if (e.errNo == EEXIST) continue;
             throw;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 00fcb9b79..b302d6f45 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -256,14 +256,12 @@ inline Paths createDirs(PathView path)
 /**
  * Create a symlink.
  */
-void createSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime = {});
+void createSymlink(const Path & target, const Path & link);
 
 /**
  * Atomically create or replace a symlink.
  */
-void replaceSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime = {});
+void replaceSymlink(const Path & target, const Path & link);
 
 void renameFile(const Path & src, const Path & dst);
 

From a0c4d585494ab80e9d0095a54cc46eb9e3977f2d Mon Sep 17 00:00:00 2001
From: Michal Sojka <michal.sojka@cvut.cz>
Date: Sun, 11 Jun 2023 17:03:14 +0200
Subject: [PATCH 084/183] Remove RegisterPrimOp constructor without support for
 documentation

The remaining constructor RegisterPrimOp::RegisterPrimOp(Info && info)
allows specifying the documentation in .args and .doc members of the
Info structure.

Commit 8ec1ba02109e removed all uses of the removed constructor in the
nix binary. Here, we remove the constructor completely as well as its
use in a plugin test. According to #8515, we didn't promis to maintain
compatibility with external plugins.

Fixes #8515
---
 src/libexpr/primops.cc      | 12 ------------
 src/libexpr/primops.hh      |  5 -----
 tests/plugins/plugintest.cc |  6 +++++-
 3 files changed, 5 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 47d8076b1..5b2f7e8b7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4058,18 +4058,6 @@ static RegisterPrimOp primop_splitVersion({
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
-{
-    if (!primOps) primOps = new PrimOps;
-    primOps->push_back({
-        .name = name,
-        .args = {},
-        .arity = arity,
-        .fun = fun,
-    });
-}
-
-
 RegisterPrimOp::RegisterPrimOp(Info && info)
 {
     if (!primOps) primOps = new PrimOps;
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 4ae73fe1f..73b7b866c 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -28,11 +28,6 @@ struct RegisterPrimOp
      * will get called during EvalState initialization, so there
      * may be primops not yet added and builtins is not yet sorted.
      */
-    RegisterPrimOp(
-        std::string name,
-        size_t arity,
-        PrimOpFun fun);
-
     RegisterPrimOp(Info && info);
 };
 
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc
index 04b791021..e02fd68d5 100644
--- a/tests/plugins/plugintest.cc
+++ b/tests/plugins/plugintest.cc
@@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args
         v.mkBool(false);
 }
 
-static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull);
+static RegisterPrimOp rp({
+    .name = "anotherNull",
+    .arity = 0,
+    .fun = prim_anotherNull,
+});

From a1cf16563f681b5cb3026f2bbca629996ed36d86 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 14 Jun 2023 22:49:58 +0200
Subject: [PATCH 085/183] Fixup description of substituters (#8291)

Introduce what substituters "are" in the configuration option entry.
Remove arbitrary line breaks for easier editing in the future.
Link glossary some more.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: John Ericson <git@JohnEricson.me>
---
 doc/manual/src/glossary.md |  7 ++++---
 src/libstore/globals.hh    | 17 ++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index e142bd415..47a484826 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -112,9 +112,10 @@
     from some server.
 
   - [substituter]{#gloss-substituter}\
-    A *substituter* is an additional store from which Nix will
-    copy store objects it doesn't have.  For details, see the
-    [`substituters` option](./command-ref/conf-file.md#conf-substituters).
+    An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them.
+    Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter.
+
+    See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details.
 
     [substituter]: #gloss-substituter
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 940cb48f5..820898350 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -691,20 +691,19 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
-          to be used as substituters, separated by whitespace.
-          Substituters are tried based on their Priority value, which each substituter can set
-          independently. Lower value means higher priority.
-          The default is `https://cache.nixos.org`, with a Priority of 40.
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
+          A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
-          At least one of the following conditions must be met for Nix to use
-          a substituter:
+          Substituters are tried based on their priority value, which each substituter can set independently.
+          Lower value means higher priority.
+          The default is `https://cache.nixos.org`, which has a priority of 40.
+
+          At least one of the following conditions must be met for Nix to use a substituter:
 
           - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
           - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
 
-          In addition, each store path should be trusted as described
-          in [`trusted-public-keys`](#conf-trusted-public-keys)
+          In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
         )",
         {"binary-caches"}};
 

From ca5752d4faecad8b87ba59892b9ca2a3d8350837 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 16 May 2023 11:21:50 -0400
Subject: [PATCH 086/183] Add another case to the `nix-collect-garbage -d` test

---
 tests/gc.sh | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/tests/gc.sh b/tests/gc.sh
index 95669e25c..9d92ad758 100644
--- a/tests/gc.sh
+++ b/tests/gc.sh
@@ -56,25 +56,34 @@ testCollectGarbageD () {
     clearProfiles
     # Run two `nix-env` commands, should create two generations of
     # the profile
-    nix-env -f ./user-envs.nix -i foo-1.0
-    nix-env -f ./user-envs.nix -i foo-2.0pre1
-    [[ $(nix-env --list-generations | wc -l) -eq 2 ]]
+    nix-env -f ./user-envs.nix -i foo-1.0 "$@"
+    nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@"
+    [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]]
 
     # Clear the profile history. There should be only one generation
     # left
     nix-collect-garbage -d
-    [[ $(nix-env --list-generations | wc -l) -eq 1 ]]
+    [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]]
 }
+
 # `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
 # using them
 if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
     testCollectGarbageD
 
+    # Run the same test, but forcing the profiles an arbitrary location.
+    rm ~/.nix-profile
+    ln -s $TEST_ROOT/blah ~/.nix-profile
+    testCollectGarbageD
+
     # Run the same test, but forcing the profiles at their legacy location under
     # /nix/var/nix.
     #
+    # Note that we *don't* use the default profile; `nix-collect-garbage` will
+    # need to check the legacy conditional unconditionally not just follow
+    # `~/.nix-profile` to pass this test.
+    #
     # Regression test for #8294
     rm ~/.nix-profile
-    ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile
-    testCollectGarbageD
+    testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me"
 fi

From d4a2ced9cb99253a277c1507baf001d51871842f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 16 May 2023 11:25:45 -0400
Subject: [PATCH 087/183] Split out `nix-collect-garbage -d` test to new file

Good for test parallelism, and separation of concerns (core GC vs
profiles deleting).
---
 tests/gc.sh                    | 37 -------------------------------
 tests/local.mk                 |  1 +
 tests/nix-collect-garbage-d.sh | 40 ++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 37 deletions(-)
 create mode 100644 tests/nix-collect-garbage-d.sh

diff --git a/tests/gc.sh b/tests/gc.sh
index 9d92ad758..ad09a8b39 100644
--- a/tests/gc.sh
+++ b/tests/gc.sh
@@ -50,40 +50,3 @@ if test -e $outPath/foobar; then false; fi
 # Check that the store is empty.
 rmdir $NIX_STORE_DIR/.links
 rmdir $NIX_STORE_DIR
-
-## Test `nix-collect-garbage -d`
-testCollectGarbageD () {
-    clearProfiles
-    # Run two `nix-env` commands, should create two generations of
-    # the profile
-    nix-env -f ./user-envs.nix -i foo-1.0 "$@"
-    nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@"
-    [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]]
-
-    # Clear the profile history. There should be only one generation
-    # left
-    nix-collect-garbage -d
-    [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]]
-}
-
-# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
-# using them
-if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
-    testCollectGarbageD
-
-    # Run the same test, but forcing the profiles an arbitrary location.
-    rm ~/.nix-profile
-    ln -s $TEST_ROOT/blah ~/.nix-profile
-    testCollectGarbageD
-
-    # Run the same test, but forcing the profiles at their legacy location under
-    # /nix/var/nix.
-    #
-    # Note that we *don't* use the default profile; `nix-collect-garbage` will
-    # need to check the legacy conditional unconditionally not just follow
-    # `~/.nix-profile` to pass this test.
-    #
-    # Regression test for #8294
-    rm ~/.nix-profile
-    testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me"
-fi
diff --git a/tests/local.mk b/tests/local.mk
index 9e340e2e2..2da47d243 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -16,6 +16,7 @@ nix_tests = \
   flakes/flake-in-submodule.sh \
   ca/gc.sh \
   gc.sh \
+  nix-collect-garbage-d.sh \
   remote-store.sh \
   legacy-ssh-store.sh \
   lang.sh \
diff --git a/tests/nix-collect-garbage-d.sh b/tests/nix-collect-garbage-d.sh
new file mode 100644
index 000000000..bf30f8938
--- /dev/null
+++ b/tests/nix-collect-garbage-d.sh
@@ -0,0 +1,40 @@
+source common.sh
+
+clearStore
+
+## Test `nix-collect-garbage -d`
+
+# TODO make `nix-env` doesn't work with CA derivations, and make
+# `ca/nix-collect-garbage-d.sh` wrapper.
+
+testCollectGarbageD () {
+    clearProfiles
+    # Run two `nix-env` commands, should create two generations of
+    # the profile
+    nix-env -f ./user-envs.nix -i foo-1.0 "$@"
+    nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@"
+    [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]]
+
+    # Clear the profile history. There should be only one generation
+    # left
+    nix-collect-garbage -d
+    [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]]
+}
+
+testCollectGarbageD
+
+# Run the same test, but forcing the profiles an arbitrary location.
+rm ~/.nix-profile
+ln -s $TEST_ROOT/blah ~/.nix-profile
+testCollectGarbageD
+
+# Run the same test, but forcing the profiles at their legacy location under
+# /nix/var/nix.
+#
+# Note that we *don't* use the default profile; `nix-collect-garbage` will
+# need to check the legacy conditional unconditionally not just follow
+# `~/.nix-profile` to pass this test.
+#
+# Regression test for #8294
+rm ~/.nix-profile
+testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me"

From b55f26c65f4ca0e80bd0f3c6436333f054acf6d6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 16 May 2023 12:41:39 -0400
Subject: [PATCH 088/183] Improve `nix-env --delete-generations` docs

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .../command-ref/nix-env/delete-generations.md | 59 ++++++++++++++++---
 1 file changed, 50 insertions(+), 9 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md
index 92cb7f0d9..d828a5b9e 100644
--- a/doc/manual/src/command-ref/nix-env/delete-generations.md
+++ b/doc/manual/src/command-ref/nix-env/delete-generations.md
@@ -9,14 +9,39 @@
 # Description
 
 This operation deletes the specified generations of the current profile.
-The generations can be a list of generation numbers, the special value
-`old` to delete all non-current generations, a value such as `30d` to
-delete all generations older than the specified number of days (except
-for the generation that was active at that point in time), or a value
-such as `+5` to keep the last `5` generations ignoring any newer than
-current, e.g., if `30` is the current generation `+5` will delete
-generation `25` and all older generations. Periodically deleting old
-generations is important to make garbage collection effective.
+
+*generations* can be a one of the following:
+
+- <span id="generations-list">`<number>...`</span>:\
+  A list of generation numbers, each one a separate command-line argument.
+
+  Delete exactly the profile generations given by their generation number.
+  Deleting the current generation is not allowed.
+
+- The special value <span id="generations-old">`old`</span>
+
+  Delete all generations older than the current one.
+
+- <span id="generations-days">`<days>d`</span>:\
+  The last *days* days
+
+  *Example*: `30d`
+
+  Delete all generations older than *days* days.
+  The generation that was active at that point in time is excluded, and will not be deleted.
+
+- <span id="generations-count">`+<count>`</span>:\
+  The last *count* generations up to the present
+
+  *Example*: `+5`
+
+  Keep the last *count* generations, along with any newer than current.
+
+Periodically deleting old generations is important to make garbage collection
+effective.
+The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion.
+
+[store object]: @docroot@/glossary.md#gloss-store-object
 
 {{#include ./opt-common.md}}
 
@@ -28,19 +53,35 @@ generations is important to make garbage collection effective.
 
 # Examples
 
+## Delete explicit generation numbers
+
 ```console
 $ nix-env --delete-generations 3 4 8
 ```
 
+Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those.
+
+## Keep most-recent by count count
+
 ```console
 $ nix-env --delete-generations +5
 ```
 
+Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`.
+
+Then this command will delete generations `20` through `25` (`<= 30 - 5`),
+and keep generations `26` through `31` (`> 30 - 5`).
+
+## Keep most-recent in days
+
 ```console
 $ nix-env --delete-generations 30d
 ```
 
+This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists).
+
+## Delete all older
+
 ```console
 $ nix-env --profile other_profile --delete-generations old
 ```
-

From 5b7e285727e23135a65597bcaab6d043d6d61ed6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 16 May 2023 10:32:38 -0400
Subject: [PATCH 089/183] Improve `nix-collect-garbage` docs

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .../src/command-ref/nix-collect-garbage.md    | 67 +++++++++++++++----
 1 file changed, 55 insertions(+), 12 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index 51db5fc67..a679ceaf7 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -1,6 +1,6 @@
 # Name
 
-`nix-collect-garbage` - delete unreachable store paths
+`nix-collect-garbage` - delete unreachable [store objects]
 
 # Synopsis
 
@@ -8,17 +8,57 @@
 
 # Description
 
-The command `nix-collect-garbage` is mostly an alias of [`nix-store
---gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all
-unreachable paths in the Nix store to clean up your system. However,
-it provides two additional options: `-d` (`--delete-old`), which
-deletes all old generations of all profiles in `/nix/var/nix/profiles`
-by invoking `nix-env --delete-generations old` on all profiles (of
-course, this makes rollbacks to previous configurations impossible);
-and `--delete-older-than` *period*, where period is a value such as
-`30d`, which deletes all generations older than the specified number
-of days in all profiles in `/nix/var/nix/profiles` (except for the
-generations that were active at that point in time).
+The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md).
+That is, it deletes all unreachable [store objects] in the Nix store to clean up your system.
+
+However, it provides two additional options,
+[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than),
+which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots.
+These options are the equivalent of running
+[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md)
+with various augments on multiple profiles,
+prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags.
+
+> **Note**
+>
+> Deleting previous configurations makes rollbacks to them impossible.
+
+These flags should be used with care, because they potentially delete generations of profiles used by other users on the system.
+
+## Locations searched for profiles
+
+`nix-collect-garbage` cannot know about all profiles; that information doesn't exist.
+Instead, it looks in a few locations, and acts on all profiles it finds there:
+
+1. The default profile locations as specified in the [profiles] section of the manual.
+
+2. > **NOTE**
+   >
+   > Not stable; subject to change
+   >
+   > Do not rely on this functionality; it just exists for migration purposes and is may change in the future.
+   > These deprecated paths remain a private implementation detail of Nix.
+
+   `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`.
+
+   With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands.
+   `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix.
+
+# Options
+
+These options are for deleting old [profiles] prior to deleting unreachable [store objects].
+
+- <span id="opt-delete-old">[`--delete-old`](#opt-delete-old)</span> / `-d`\
+  Delete all old generations of profiles.
+
+  This is the equivalent of invoking `nix-env --delete-generations old` on each found profile.
+
+- <span id="opt-delete-older-than">[`--delete-older-than`](#opt-delete-older-than)</span> *period*\
+  Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time).
+  *period* is a value such as `30d`, which would mean 30 days.
+
+  This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile.
+  See the documentation of that command for additional information about the *period* argument.
 
 {{#include ./opt-common.md}}
 
@@ -32,3 +72,6 @@ generations of each profile, do
 ```console
 $ nix-collect-garbage -d
 ```
+
+[profiles]: @docroot@/command-ref/files/profiles.md
+[store objects]: @docroot@/glossary.md#gloss-store-object

From 4b487317c30060024df436d010e6126be3a3d49e Mon Sep 17 00:00:00 2001
From: scarf <greenscarf005@gmail.com>
Date: Thu, 15 Jun 2023 08:52:34 +0900
Subject: [PATCH 090/183] style: use mathematical interval notation

---
 scripts/install-multi-user.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 2e8f2b8c3..45885bfd9 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -700,8 +700,8 @@ EOF
 }
 
 welcome_to_nix() {
-    local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
-    local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}")
+    local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
+    local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}")
     local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}")
 
     ok "Welcome to the Multi-User Nix Installation"

From e09b40e0d0b68ca7c3646ddffb50e1356daec997 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sun, 5 Mar 2023 02:36:26 +0100
Subject: [PATCH 091/183] reword documentation on trusted users and
 substituters

this is to make it slightly easier to scan over
---
 src/libstore/globals.hh | 14 ++++++--------
 src/nix/daemon.cc       | 30 +++++++++++++++---------------
 2 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 820898350..b0c025c8e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -700,8 +700,8 @@ public:
 
           At least one of the following conditions must be met for Nix to use a substituter:
 
-          - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
-          - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
+          - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
+          - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
 
           In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
         )",
@@ -710,12 +710,10 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format),
-          separated by whitespace. These are
-          not used by default, but can be enabled by users of the Nix daemon
-          by specifying `--option substituters urls` on the command
-          line. Unprivileged users are only allowed to pass a subset of the
-          URLs listed in `substituters` and `trusted-substituters`.
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters).
+
+          Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`.
         )",
         {"trusted-binary-caches"}};
 
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 9fe9b3b1e..09adab5d3 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -55,19 +55,16 @@ struct AuthorizationSettings : Config {
     Setting<Strings> trustedUsers{
         this, {"root"}, "trusted-users",
         R"(
-          A list of names of users (separated by whitespace) that have
-          additional rights when connecting to the Nix daemon, such as the
-          ability to specify additional binary caches, or to import unsigned
-          NARs. You can also specify groups by prefixing them with `@`; for
-          instance, `@wheel` means all users in the `wheel` group. The default
-          is `root`.
+          A list of user names, separated by whitespace.
+          These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s.
+
+          You can also specify groups by prefixing names with `@`.
+          For instance, `@wheel` means all users in the `wheel` group.
 
           > **Warning**
           >
-          > Adding a user to `trusted-users` is essentially equivalent to
-          > giving that user root access to the system. For example, the user
-          > can set `sandbox-paths` and thereby obtain read access to
-          > directories that are otherwise inacessible to them.
+          > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system.
+          > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them.
         )"};
 
     /**
@@ -76,12 +73,15 @@ struct AuthorizationSettings : Config {
     Setting<Strings> allowedUsers{
         this, {"*"}, "allowed-users",
         R"(
-          A list of names of users (separated by whitespace) that are allowed
-          to connect to the Nix daemon. As with the `trusted-users` option,
-          you can specify groups by prefixing them with `@`. Also, you can
-          allow all users by specifying `*`. The default is `*`.
+          A list user names, separated by whitespace.
+          These users are allowed to connect to the Nix daemon.
 
-          Note that trusted users are always allowed to connect.
+          As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`.
+          Also, you can allow all users by specifying `*`.
+
+          > **Note**
+          >
+          > Trusted users are always allowed to connect to the Nix daemon.
         )"};
 };
 

From c453719d6e9e1f96f4591b3da359d08b0ce11575 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 15 Jun 2023 02:29:30 +0200
Subject: [PATCH 092/183] rename files referring to antiquotation

since we renamed this to string interpolation, file names should be
fixed up as well
---
 ...d-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} | 0
 ...d-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} | 0
 ...d-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} | 0
 ...-antiquotation.exp => eval-okay-path-string-interpolation.exp} | 0
 ...-antiquotation.nix => eval-okay-path-string-interpolation.nix} | 0
 5 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/lang/{eval-fail-bad-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} (100%)
 rename tests/lang/{eval-fail-bad-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} (100%)
 rename tests/lang/{eval-fail-bad-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} (100%)
 rename tests/lang/{eval-okay-path-antiquotation.exp => eval-okay-path-string-interpolation.exp} (100%)
 rename tests/lang/{eval-okay-path-antiquotation.nix => eval-okay-path-string-interpolation.nix} (100%)

diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/lang/eval-fail-bad-string-interpolation-1.nix
similarity index 100%
rename from tests/lang/eval-fail-bad-antiquote-1.nix
rename to tests/lang/eval-fail-bad-string-interpolation-1.nix
diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/lang/eval-fail-bad-string-interpolation-2.nix
similarity index 100%
rename from tests/lang/eval-fail-bad-antiquote-2.nix
rename to tests/lang/eval-fail-bad-string-interpolation-2.nix
diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/lang/eval-fail-bad-string-interpolation-3.nix
similarity index 100%
rename from tests/lang/eval-fail-bad-antiquote-3.nix
rename to tests/lang/eval-fail-bad-string-interpolation-3.nix
diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/lang/eval-okay-path-string-interpolation.exp
similarity index 100%
rename from tests/lang/eval-okay-path-antiquotation.exp
rename to tests/lang/eval-okay-path-string-interpolation.exp
diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/lang/eval-okay-path-string-interpolation.nix
similarity index 100%
rename from tests/lang/eval-okay-path-antiquotation.nix
rename to tests/lang/eval-okay-path-string-interpolation.nix

From 520491607efb7b65704b600077e8fed21562597a Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 15 Jun 2023 02:31:47 +0200
Subject: [PATCH 093/183] docs issue template: move checklist down

it's annoying to write issues with the checklist in the way, and the
proposal is more important.
---
 .github/ISSUE_TEMPLATE/missing_documentation.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md
index 942d7a971..be3f6af97 100644
--- a/.github/ISSUE_TEMPLATE/missing_documentation.md
+++ b/.github/ISSUE_TEMPLATE/missing_documentation.md
@@ -11,6 +11,10 @@ assignees: ''
 
 <!-- describe your problem -->
 
+## Proposal
+
+<!-- propose a solution -->
+
 ## Checklist
 
 <!-- make sure this issue is not redundant or obsolete -->
@@ -22,10 +26,6 @@ assignees: ''
 [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src
 [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation
 
-## Proposal
-
-<!-- propose a solution -->
-
 ## Priorities
 
 Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc).

From c3c40763421e881ed5c326ed88a0539b6585c368 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 15 Jun 2023 03:06:38 +0200
Subject: [PATCH 094/183] make domain-specificity more specific

also slightly reword the purpose statement to introduce (and explain)
derivations right away.
---
 doc/manual/src/language/index.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 69ebc70c0..29950a52d 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -1,10 +1,11 @@
 # Nix Language
 
-The Nix language is
+The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files.
+It is:
 
 - *domain-specific*
 
-  Its purpose is to conveniently create and compose precise descriptions of how contents of existing files are used to derive new files.
+  It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language.
 
 - *declarative*
 

From b7d47e1d22e7ce2785487d325cc3dd35a43f16b5 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 15 Jun 2023 04:58:07 +0200
Subject: [PATCH 095/183] fix wording

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b0c025c8e..f63ec8b50 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -713,7 +713,7 @@ public:
           A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
           These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters).
 
-          Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`.
+          Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`.
         )",
         {"trusted-binary-caches"}};
 

From 098fbf6273bb5ee69b8dc1a4354d45d9a605541a Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Wed, 14 Jun 2023 21:47:58 -0700
Subject: [PATCH 096/183] src/libexpr/eval.hh: fix typo

The option name is `allowed-uris`, not `allowed-uri`.
---
 src/libexpr/eval.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index d6f4560a5..ea1f5975b 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -741,7 +741,7 @@ struct EvalSettings : Config
           If set to `true`, the Nix evaluator will not allow access to any
           files outside of the Nix search path (as set via the `NIX_PATH`
           environment variable or the `-I` option), or to URIs outside of
-          `allowed-uri`. The default is `false`.
+          `allowed-uris`. The default is `false`.
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",

From 80451b762d71bfbc14dbd2c2d88fda7dcb9571cf Mon Sep 17 00:00:00 2001
From: scarf <greenscarf005@gmail.com>
Date: Thu, 15 Jun 2023 14:47:18 +0900
Subject: [PATCH 097/183] style: use plurals in uid ranges

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 45885bfd9..656769d84 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -701,7 +701,7 @@ EOF
 
 welcome_to_nix() {
     local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
-    local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}")
+    local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}")
     local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}")
 
     ok "Welcome to the Multi-User Nix Installation"

From e1fa48f17c0bb0f73e97e684077cec8dfa1d7a3d Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Thu, 15 Jun 2023 07:41:37 -0400
Subject: [PATCH 098/183] Update src/nix/daemon.cc

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/nix/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 09adab5d3..a57070c56 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -81,7 +81,7 @@ struct AuthorizationSettings : Config {
 
           > **Note**
           >
-          > Trusted users are always allowed to connect to the Nix daemon.
+          > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon.
         )"};
 };
 

From d2696cdd1ee404ef6c6bd532cb90007dd8ee3e9f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 14 Jun 2023 22:57:42 +0200
Subject: [PATCH 099/183] Fix build hook error for libstore library users

A library shouldn't require changes to the caller's argument handling,
especially if it doesn't have to, and indeed we don't have to.

This changes the lookup order to prioritize the hardcoded path to nix
if it exists. The static executable still finds itself through /proc
and the like.
---
 .gitignore                            |  1 +
 Makefile                              |  1 +
 src/libstore/globals.cc               | 25 ++++++++++++++-
 tests/local.mk                        |  2 ++
 tests/test-libstoreconsumer.sh        |  6 ++++
 tests/test-libstoreconsumer/README.md |  6 ++++
 tests/test-libstoreconsumer/local.mk  | 12 +++++++
 tests/test-libstoreconsumer/main.cc   | 45 +++++++++++++++++++++++++++
 8 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 tests/test-libstoreconsumer.sh
 create mode 100644 tests/test-libstoreconsumer/README.md
 create mode 100644 tests/test-libstoreconsumer/local.mk
 create mode 100644 tests/test-libstoreconsumer/main.cc

diff --git a/.gitignore b/.gitignore
index 7ae1071d0..29d9106ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,6 +89,7 @@ perl/Makefile.config
 /tests/ca/config.nix
 /tests/dyn-drv/config.nix
 /tests/repl-result-out
+/tests/test-libstoreconsumer/test-libstoreconsumer
 
 # /tests/lang/
 /tests/lang/*.out
diff --git a/Makefile b/Makefile
index d6b49473a..c6220482a 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,7 @@ makefiles += \
   src/libstore/tests/local.mk \
   src/libexpr/tests/local.mk \
   tests/local.mk \
+  tests/test-libstoreconsumer/local.mk \
   tests/plugins/local.mk
 else
 makefiles += \
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 32e9a6ea9..d53377239 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -77,7 +77,30 @@ Settings::Settings()
     allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
 #endif
 
-    buildHook = getSelfExe().value_or("nix") + " __build-remote";
+    /* Set the build hook location
+
+       For builds we perform a self-invocation, so Nix has to be self-aware.
+       That is, it has to know where it is installed. We don't think it's sentient.
+
+       Normally, nix is installed according to `nixBinDir`, which is set at compile time,
+       but can be overridden. This makes for a great default that works even if this
+       code is linked as a library into some other program whose main is not aware
+       that it might need to be a build remote hook.
+
+       However, it may not have been installed at all. For example, if it's a static build,
+       there's a good chance that it has been moved out of its installation directory.
+       That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the
+       current executable, using `getSelfExe()`.
+
+       As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible.
+       If you're porting Nix to a new platform, that might be good enough for a while, but
+       you'll want to improve `getSelfExe()` to work on your platform.
+     */
+    std::string nixExePath = nixBinDir + "/nix";
+    if (!pathExists(nixExePath)) {
+        nixExePath = getSelfExe().value_or("nix");
+    }
+    buildHook = nixExePath + " __build-remote";
 }
 
 void loadConfFile()
diff --git a/tests/local.mk b/tests/local.mk
index 9e340e2e2..a37365efe 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -134,6 +134,7 @@ nix_tests = \
   flakes/show.sh \
   impure-derivations.sh \
   path-from-hash-part.sh \
+  test-libstoreconsumer.sh \
   toString-path.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
@@ -152,6 +153,7 @@ test-deps += \
   tests/common/vars-and-functions.sh \
   tests/config.nix \
   tests/ca/config.nix \
+  tests/test-libstoreconsumer/test-libstoreconsumer \
   tests/dyn-drv/config.nix
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
diff --git a/tests/test-libstoreconsumer.sh b/tests/test-libstoreconsumer.sh
new file mode 100644
index 000000000..8a77cf5a1
--- /dev/null
+++ b/tests/test-libstoreconsumer.sh
@@ -0,0 +1,6 @@
+source common.sh
+
+drv="$(nix-instantiate simple.nix)"
+cat "$drv"
+out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")"
+cat "$out/hello" | grep -F "Hello World!"
diff --git a/tests/test-libstoreconsumer/README.md b/tests/test-libstoreconsumer/README.md
new file mode 100644
index 000000000..ded69850f
--- /dev/null
+++ b/tests/test-libstoreconsumer/README.md
@@ -0,0 +1,6 @@
+
+A very simple C++ consumer of the libstore library.
+
+  - Keep it simple. Library consumers expect something simple.
+  - No build hook, or any other reinvocations.
+  - No more global state than necessary.
diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk
new file mode 100644
index 000000000..cd2d0c7f8
--- /dev/null
+++ b/tests/test-libstoreconsumer/local.mk
@@ -0,0 +1,12 @@
+programs += test-libstoreconsumer
+
+test-libstoreconsumer_DIR := $(d)
+
+test-libstoreconsumer_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore
+
+test-libstoreconsumer_LIBS = libstore libutil
+
+test-libstoreconsumer_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc
new file mode 100644
index 000000000..31b6d8ef1
--- /dev/null
+++ b/tests/test-libstoreconsumer/main.cc
@@ -0,0 +1,45 @@
+#include "globals.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+#include <iostream>
+
+using namespace nix;
+
+int main (int argc, char **argv)
+{
+    try {
+        if (argc != 2) {
+            std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n";
+            return 1;
+        }
+
+        std::string drvPath = argv[1];
+
+        initLibStore();
+
+        auto store = nix::openStore();
+
+        // build the derivation
+
+        std::vector<DerivedPath> paths {
+            DerivedPath::Built {
+                .drvPath = store->parseStorePath(drvPath),
+                .outputs = OutputsSpec::Names{"out"}
+            }
+        };
+
+        const auto results = store->buildPathsWithResults(paths, bmNormal, store);
+
+        for (const auto & result : results) {
+            for (const auto & [outputName, realisation] : result.builtOutputs) {
+                std::cout << store->printStorePath(realisation.outPath) << "\n";
+            }
+        }
+
+        return 0;
+
+    } catch (const std::exception & e) {
+        std::cerr << "Error: " << e.what() << "\n";
+        return 1;
+    }
+}

From 2ceacce484e21ac116a79c74877327355fd153d0 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 15 Jun 2023 15:57:54 +0200
Subject: [PATCH 100/183] Update src/libstore/globals.hh

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index f63ec8b50..46147a5e1 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -713,7 +713,7 @@ public:
           A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
           These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters).
 
-          Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`.
+          Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.
         )",
         {"trusted-binary-caches"}};
 

From b2247ef4f6b02cf4e666455eb9afc86398fff35d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Thu, 15 Jun 2023 21:24:14 +0200
Subject: [PATCH 101/183] Don't assume the type of string::size_type

The code accidentally conflated `std::string::size_type` and `long unsigned int`.
This was fine on 64bits machines where they are apparently the same in
practice, but not on 32bits. Fix that by using `std::string::size_type`
everywhere.
---
 src/libutil/references.cc | 2 +-
 src/libutil/references.hh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index 74003584a..7f59b4c09 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -75,7 +75,7 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S
 RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink)
     : rewrites(rewrites), nextSink(nextSink)
 {
-    long unsigned int maxRewriteSize = 0;
+    std::string::size_type maxRewriteSize = 0;
     for (auto & [from, to] : rewrites) {
         assert(from.size() == to.size());
         maxRewriteSize = std::max(maxRewriteSize, from.size());
diff --git a/src/libutil/references.hh b/src/libutil/references.hh
index ffd730e7b..f0baeffe1 100644
--- a/src/libutil/references.hh
+++ b/src/libutil/references.hh
@@ -26,7 +26,7 @@ public:
 struct RewritingSink : Sink
 {
     const StringMap rewrites;
-    long unsigned int maxRewriteSize;
+    std::string::size_type maxRewriteSize;
     std::string prev;
     Sink & nextSink;
     uint64_t pos = 0;

From 4a33d5fe3549137bacf9373e5ba7bfe11a421099 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 16 Jun 2023 14:33:31 +0200
Subject: [PATCH 102/183] fix link text

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/nix/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index a57070c56..d0e70a7b1 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -56,7 +56,7 @@ struct AuthorizationSettings : Config {
         this, {"root"}, "trusted-users",
         R"(
           A list of user names, separated by whitespace.
-          These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s.
+          These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NARs](@docroot@/glossary.md#gloss-nar).
 
           You can also specify groups by prefixing names with `@`.
           For instance, `@wheel` means all users in the `wheel` group.

From 1a8ca85d488ddacf26f2aeddddab926c0e081d98 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 16 Jun 2023 14:34:11 +0200
Subject: [PATCH 103/183] use "store URLs" consistently

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 46147a5e1..d2efd1505 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -710,7 +710,7 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
           These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.

From 126eea48e300ab365c46ce062776e74a3907a7c8 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 16 Jun 2023 14:36:53 +0200
Subject: [PATCH 104/183] do not refer to `trusted-users` another time

---
 src/nix/daemon.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index d0e70a7b1..43d2f8f86 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -76,7 +76,8 @@ struct AuthorizationSettings : Config {
           A list user names, separated by whitespace.
           These users are allowed to connect to the Nix daemon.
 
-          As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`.
+          You can specify groups by prefixing names with `@`.
+          For instance, `@wheel` means all users in the `wheel` group.
           Also, you can allow all users by specifying `*`.
 
           > **Note**

From baef05e6fefb46b3bdddb2785861bd7190920506 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 16 Jun 2023 14:37:08 +0200
Subject: [PATCH 105/183] fix typo

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/nix/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 43d2f8f86..6dcd7f94d 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -82,7 +82,7 @@ struct AuthorizationSettings : Config {
 
           > **Note**
           >
-          > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon.
+          > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always connect to the Nix daemon.
         )"};
 };
 

From cab03fb7794f6e20e02dc1460a78201ab3955c18 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 16 Jun 2023 15:58:42 +0200
Subject: [PATCH 106/183] Add docs

---
 doc/manual/src/SUMMARY.md.in                |  2 ++
 doc/manual/src/protocols/protocols.md       |  4 +++
 doc/manual/src/protocols/tarball-fetcher.md | 40 +++++++++++++++++++++
 3 files changed, 46 insertions(+)
 create mode 100644 doc/manual/src/protocols/protocols.md
 create mode 100644 doc/manual/src/protocols/tarball-fetcher.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 69c721b57..cfbb38be8 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -98,6 +98,8 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture](architecture/architecture.md)
+- [Protocols](protocols/protocols.md)
+  - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)
diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/protocols.md
new file mode 100644
index 000000000..d6bf1d809
--- /dev/null
+++ b/doc/manual/src/protocols/protocols.md
@@ -0,0 +1,4 @@
+# Protocols
+
+This chapter documents various developer-facing interfaces provided by
+Nix.
diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md
new file mode 100644
index 000000000..1c6c92000
--- /dev/null
+++ b/doc/manual/src/protocols/tarball-fetcher.md
@@ -0,0 +1,40 @@
+# Serving Tarball Flakes
+
+Tarball flakes are served as regular tarballs via HTTP or the file
+system (for `file://` URLs).
+
+An HTTP server can return an "immutable" flakeref appropriate for lock
+files. This allows users to specify a tarball flake input in
+`flake.nix` that requests the latest version of a flake
+(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock`
+will record a URL whose contents will not change
+(e.g. `https://example.org/hello/<revision>.tar.gz`). To do so, the
+server must return a `Link` header with the `rel` attribute set to
+`immutable`, as follows:
+
+```
+Link: <flakeref>; rel="immutable"
+```
+
+(Note the required `<` and `>` characters around *flakeref*.)
+
+*flakeref* must be a tarball flakeref. It can contain flake attributes
+such as `narHash`, `rev` and `revCount`. If `narHash` is included, its
+value must be the NAR hash of the unpacked tarball (as computed via
+`nix hash path`). Nix checks the contents of the returned tarball
+against the `narHash` attribute. The `rev` and `revCount` attributes
+are useful when the tarball flake is a mirror of a fetcher type that
+has those attributes, such as Git or GitHub. They are not checked by
+Nix.
+
+```
+Link: <https://example.org/hello/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz
+  ?rev=442793d9ec0584f6a6e82fa253850c8085bb150a
+  &revCount=835
+  &narHash=sha256-GUm8Uh/U74zFCwkvt9Mri4DSM%2BmHj3tYhXUkYpiv31M%3D>; rel="immutable"
+```
+
+(The linebreaks in this example are for clarity and must not be included in the actual response.)
+
+For tarball flakes, the value of the `lastModified` flake attribute is
+defined as the timestamp of the newest file inside the tarball.

From b1ed9b4b0cc037a8b14dcc37fd32f6a5a16e7ba3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 16 Jun 2023 16:48:37 +0200
Subject: [PATCH 107/183] Apply suggestions from code review

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/protocols/tarball-fetcher.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md
index 1c6c92000..0d3212303 100644
--- a/doc/manual/src/protocols/tarball-fetcher.md
+++ b/doc/manual/src/protocols/tarball-fetcher.md
@@ -1,15 +1,17 @@
-# Serving Tarball Flakes
+# Lockable HTTP Tarball Protocol
 
-Tarball flakes are served as regular tarballs via HTTP or the file
-system (for `file://` URLs).
+Tarball flakes can be served as regular tarballs via HTTP or the file
+system (for `file://` URLs). Unless the server implements the Lockable
+HTTP Tarball protocol, it is the responsibility of the user to make sure that
+the URL always produces the same tarball contents.
 
-An HTTP server can return an "immutable" flakeref appropriate for lock
+An HTTP server can return an "immutable" HTTP URL appropriate for lock
 files. This allows users to specify a tarball flake input in
 `flake.nix` that requests the latest version of a flake
 (e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock`
 will record a URL whose contents will not change
 (e.g. `https://example.org/hello/<revision>.tar.gz`). To do so, the
-server must return a `Link` header with the `rel` attribute set to
+server must return an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to
 `immutable`, as follows:
 
 ```

From f695a74751c314cc426ff7bbc67ce5de8b58bbfd Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 16 Jun 2023 17:58:01 +0200
Subject: [PATCH 108/183] Update src/libstore/globals.hh

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d2efd1505..05aa8288a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -711,7 +711,7 @@ public:
         this, {}, "trusted-substituters",
         R"(
           A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
-          These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters).
+          These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.
         )",

From 741f7837f85d1675efbcf43877bbcdfb9a862745 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christina=20S=C3=B8rensen?=
 <89321978+cafkafk@users.noreply.github.com>
Date: Sat, 17 Jun 2023 09:06:17 +0000
Subject: [PATCH 109/183] Fix wikipedia links (#8533)

---
 doc/manual/src/architecture/architecture.md | 6 +++---
 doc/manual/src/contributing/hacking.md      | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md
index e51958052..9e969972e 100644
--- a/doc/manual/src/architecture/architecture.md
+++ b/doc/manual/src/architecture/architecture.md
@@ -7,11 +7,11 @@ It should help users understand why Nix behaves as it does, and it should help d
 
 Nix consists of [hierarchical layers].
 
-[hierarchical layers]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers
+[hierarchical layers]: https://en.wikipedia.org/wiki/Multitier_architecture#Layers
 
 The following [concept map] shows its main components (rectangles), the objects they operate on (rounded rectangles), and their interactions (connecting phrases):
 
-[concept map]: https://en.m.wikipedia.org/wiki/Concept_map
+[concept map]: https://en.wikipedia.org/wiki/Concept_map
 
 ```
 
@@ -76,7 +76,7 @@ The result of a build task can be input to another build task.
 The following [data flow diagram] shows a build plan for illustration.
 Build inputs used as instructions to a build task are marked accordingly:
 
-[data flow diagram]: https://en.m.wikipedia.org/wiki/Data-flow_diagram
+[data flow diagram]: https://en.wikipedia.org/wiki/Data-flow_diagram
 
 ```
 +--------------------------------------------------------------------+
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index b954a2167..c57d45138 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -378,7 +378,7 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman
 [`mdbook-linkcheck`] does not implement checking [URI fragments] yet.
 
 [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck
-[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment
+[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment
 
 #### `@docroot@` variable
 

From b931d83550b200676c36ae8d1038a281ae4aa0ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Na=C3=AFm=20Favier?= <n@monade.li>
Date: Sat, 17 Jun 2023 15:05:10 +0200
Subject: [PATCH 110/183] ci: bump install-nix-action, don't fail fast

---
 .github/workflows/ci.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0f1f6d43f..c3a17d106 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,6 +11,7 @@ jobs:
   tests:
     needs: [check_secrets]
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
@@ -19,7 +20,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v21
+    - uses: cachix/install-nix-action@v22
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -61,7 +62,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v21
+    - uses: cachix/install-nix-action@v22
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
@@ -76,13 +77,14 @@ jobs:
     needs: [installer, check_secrets]
     if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v21
+    - uses: cachix/install-nix-action@v22
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -109,7 +111,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v21
+    - uses: cachix/install-nix-action@v22
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

From c8825e9d8c3fa802811f3829d055e3ef9aae90e2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 16 Jun 2023 15:19:14 -0400
Subject: [PATCH 111/183] Create nlohmann serializers for `std::optional` and
 use

This is somewhat tricky.
---
 src/libutil/args.cc                  | 15 ++----
 src/libutil/experimental-features.hh |  8 ++-
 src/libutil/json-utils.cc            | 19 +++++++
 src/libutil/json-utils.hh            | 80 +++++++++++++++++++++++-----
 4 files changed, 97 insertions(+), 25 deletions(-)
 create mode 100644 src/libutil/json-utils.cc

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 081dbeb28..3cf3ed9ca 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -1,10 +1,9 @@
 #include "args.hh"
 #include "hash.hh"
+#include "json-utils.hh"
 
 #include <glob.h>
 
-#include <nlohmann/json.hpp>
-
 namespace nix {
 
 void Args::addFlag(Flag && flag_)
@@ -247,11 +246,7 @@ nlohmann::json Args::toJSON()
             j["arity"] = flag->handler.arity;
         if (!flag->labels.empty())
             j["labels"] = flag->labels;
-        // TODO With C++23 use `std::optional::tranform`
-        if (auto & xp = flag->experimentalFeature)
-            j["experimental-feature"] = showExperimentalFeature(*xp);
-        else
-            j["experimental-feature"] = nullptr;
+        j["experimental-feature"] = flag->experimentalFeature;
         flags[name] = std::move(j);
     }
 
@@ -416,11 +411,7 @@ nlohmann::json MultiCommand::toJSON()
         cat["id"] = command->category();
         cat["description"] = trim(categories[command->category()]);
         j["category"] = std::move(cat);
-        // TODO With C++23 use `std::optional::tranform`
-        if (auto xp = command->experimentalFeature())
-            cat["experimental-feature"] = showExperimentalFeature(*xp);
-        else
-            cat["experimental-feature"] = nullptr;
+        cat["experimental-feature"] = command->experimentalFeature();
         cmds[name] = std::move(j);
     }
 
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 892c6c371..15ff5e0cd 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -3,7 +3,7 @@
 
 #include "comparator.hh"
 #include "error.hh"
-#include "nlohmann/json_fwd.hpp"
+#include "json-utils.hh"
 #include "types.hh"
 
 namespace nix {
@@ -93,4 +93,10 @@ public:
 void to_json(nlohmann::json &, const ExperimentalFeature &);
 void from_json(const nlohmann::json &, ExperimentalFeature &);
 
+/**
+ * It is always rendered as a string
+ */
+template<>
+struct json_avoids_null<ExperimentalFeature> : std::true_type {};
+
 }
diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc
new file mode 100644
index 000000000..d7220e71d
--- /dev/null
+++ b/src/libutil/json-utils.cc
@@ -0,0 +1,19 @@
+#include "json-utils.hh"
+
+namespace nix {
+
+const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+nlohmann::json * get(nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+}
diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh
index eb00e954f..5e63c1af4 100644
--- a/src/libutil/json-utils.hh
+++ b/src/libutil/json-utils.hh
@@ -2,21 +2,77 @@
 ///@file
 
 #include <nlohmann/json.hpp>
+#include <list>
 
 namespace nix {
 
-const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
-{
-    auto i = map.find(key);
-    if (i == map.end()) return nullptr;
-    return &*i;
-}
+const nlohmann::json * get(const nlohmann::json & map, const std::string & key);
 
-nlohmann::json * get(nlohmann::json & map, const std::string & key)
-{
-    auto i = map.find(key);
-    if (i == map.end()) return nullptr;
-    return &*i;
-}
+nlohmann::json * get(nlohmann::json & map, const std::string & key);
+
+/**
+ * For `adl_serializer<std::optional<T>>` below, we need to track what
+ * types are not already using `null`. Only for them can we use `null`
+ * to represent `std::nullopt`.
+ */
+template<typename T>
+struct json_avoids_null;
+
+/**
+ * Handle numbers in default impl
+ */
+template<typename T>
+struct json_avoids_null : std::bool_constant<std::is_integral<T>::value> {};
+
+template<>
+struct json_avoids_null<std::nullptr_t> : std::false_type {};
+
+template<>
+struct json_avoids_null<bool> : std::true_type {};
+
+template<>
+struct json_avoids_null<std::string> : std::true_type {};
+
+template<typename T>
+struct json_avoids_null<std::vector<T>> : std::true_type {};
+
+template<typename T>
+struct json_avoids_null<std::list<T>> : std::true_type {};
+
+template<typename K, typename V>
+struct json_avoids_null<std::map<K, V>> : std::true_type {};
+
+}
+
+namespace nlohmann {
+
+/**
+ * This "instance" is widely requested, see
+ * https://github.com/nlohmann/json/issues/1749, but momentum has stalled
+ * out. Writing there here in Nix as a stop-gap.
+ *
+ * We need to make sure the underlying type does not use `null` for this to
+ * round trip. We do that with a static assert.
+ */
+template<typename T>
+struct adl_serializer<std::optional<T>> {
+    static std::optional<T> from_json(const json & json) {
+        static_assert(
+            nix::json_avoids_null<T>::value,
+            "null is already in use for underlying type's JSON");
+        return json.is_null()
+            ? std::nullopt
+            : std::optional { adl_serializer<T>::from_json(json) };
+    }
+    static void to_json(json & json, std::optional<T> t) {
+        static_assert(
+            nix::json_avoids_null<T>::value,
+            "null is already in use for underlying type's JSON");
+        if (t)
+            adl_serializer<T>::to_json(json, *t);
+        else
+            json = nullptr;
+    }
+};
 
 }

From d2ce2e89b178e7e7467cf4c1e572704a4c2ca75e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 19 May 2023 10:56:59 -0400
Subject: [PATCH 112/183] Split `OptionalPathSetting` from `PathSetting`

Rather than doing `allowEmpty` as boolean, have separate types and use
`std::optional`. This makes it harder to forget the possibility of an
empty path.

The `build-hook` setting was categorized as a `PathSetting`, but
actually it was split into arguments. No good! Now, it is
`Setting<Strings>` which actually reflects what it means and how it is
used.

Because of the subtyping, we now also have support for
`Setting<std::optional<String>>` in general. I imagine this can be used
to clean up many more settings also.
---
 src/libstore/build/hook-instance.cc         |  6 +-
 src/libstore/build/local-derivation-goal.cc |  8 ++-
 src/libstore/globals.cc                     |  5 +-
 src/libstore/globals.hh                     |  6 +-
 src/libstore/local-fs-store.hh              | 14 ++---
 src/libstore/store-api.hh                   |  2 +-
 src/libutil/abstract-setting-to-json.hh     |  1 +
 src/libutil/config-impl.hh                  | 61 +++++++++++++++++++-
 src/libutil/config.cc                       | 63 ++++++++-------------
 src/libutil/config.hh                       | 31 ++++++++--
 10 files changed, 134 insertions(+), 63 deletions(-)

diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index 075ad554f..337c60bd4 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -5,14 +5,14 @@ namespace nix {
 
 HookInstance::HookInstance()
 {
-    debug("starting build hook '%s'", settings.buildHook);
+    debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get()));
 
-    auto buildHookArgs = tokenizeString<std::list<std::string>>(settings.buildHook.get());
+    auto buildHookArgs = settings.buildHook.get();
 
     if (buildHookArgs.empty())
         throw Error("'build-hook' setting is empty");
 
-    auto buildHook = buildHookArgs.front();
+    auto buildHook = canonPath(buildHookArgs.front());
     buildHookArgs.pop_front();
 
     Strings args;
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 7f87cdf55..cf44779b7 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -65,8 +65,9 @@ void handleDiffHook(
     const Path & tryA, const Path & tryB,
     const Path & drvPath, const Path & tmpDir)
 {
-    auto diffHook = settings.diffHook;
-    if (diffHook != "" && settings.runDiffHook) {
+    auto & diffHookOpt = settings.diffHook.get();
+    if (diffHookOpt && settings.runDiffHook) {
+        auto & diffHook = *diffHookOpt;
         try {
             auto diffRes = runProgram(RunOptions {
                 .program = diffHook,
@@ -1428,7 +1429,8 @@ void LocalDerivationGoal::startDaemon()
     Store::Params params;
     params["path-info-cache-size"] = "0";
     params["store"] = worker.store.storeDir;
-    params["root"] = getLocalStore().rootDir;
+    if (auto & optRoot = getLocalStore().rootDir.get())
+        params["root"] = *optRoot;
     params["state"] = "/no-such-path";
     params["log"] = "/no-such-path";
     auto store = make_ref<RestrictedStore>(params,
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index d53377239..5a4cb1824 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -100,7 +100,10 @@ Settings::Settings()
     if (!pathExists(nixExePath)) {
         nixExePath = getSelfExe().value_or("nix");
     }
-    buildHook = nixExePath + " __build-remote";
+    buildHook = {
+        nixExePath,
+        "__build-remote",
+    };
 }
 
 void loadConfFile()
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 820898350..b8dcf1f76 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -236,7 +236,7 @@ public:
         )",
         {"build-timeout"}};
 
-    PathSetting buildHook{this, true, "", "build-hook",
+    Setting<Strings> buildHook{this, {}, "build-hook",
         R"(
           The path to the helper program that executes remote builds.
 
@@ -556,8 +556,8 @@ public:
           line.
         )"};
 
-    PathSetting diffHook{
-        this, true, "", "diff-hook",
+    OptionalPathSetting diffHook{
+        this, std::nullopt, "diff-hook",
         R"(
           Absolute path to an executable capable of diffing build
           results. The hook is executed if `run-diff-hook` is true, and the
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index a03bb88f5..2ee2ef0c8 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -15,22 +15,22 @@ struct LocalFSStoreConfig : virtual StoreConfig
     // it to omit the call to the Setting constructor. Clang works fine
     // either way.
 
-    const PathSetting rootDir{(StoreConfig*) this, true, "",
+    const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt,
         "root",
         "Directory prefixed to all other paths."};
 
-    const PathSetting stateDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
+    const PathSetting stateDir{(StoreConfig*) this,
+        rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
         "state",
         "Directory where Nix will store state."};
 
-    const PathSetting logDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
+    const PathSetting logDir{(StoreConfig*) this,
+        rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
         "log",
         "directory where Nix will store log files."};
 
-    const PathSetting realStoreDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
+    const PathSetting realStoreDir{(StoreConfig*) this,
+        rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
         "Physical path of the Nix store."};
 };
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 2ecbe2708..14a862eef 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -114,7 +114,7 @@ struct StoreConfig : public Config
         return "";
     }
 
-    const PathSetting storeDir_{this, false, settings.nixStore,
+    const PathSetting storeDir_{this, settings.nixStore,
         "store",
         R"(
           Logical location of the Nix store, usually
diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh
index 7b6c3fcb5..d506dfb74 100644
--- a/src/libutil/abstract-setting-to-json.hh
+++ b/src/libutil/abstract-setting-to-json.hh
@@ -3,6 +3,7 @@
 
 #include <nlohmann/json.hpp>
 #include "config.hh"
+#include "json-utils.hh"
 
 namespace nix {
 template<typename T>
diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh
index b6cae5ec3..14822050e 100644
--- a/src/libutil/config-impl.hh
+++ b/src/libutil/config-impl.hh
@@ -50,8 +50,11 @@ template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set
 template<typename T>
 void BaseSetting<T>::appendOrSet(T && newValue, bool append)
 {
-    static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type");
+    static_assert(
+        !trait::appendable,
+        "using default `appendOrSet` implementation with an appendable type");
     assert(!append);
+
     value = std::move(newValue);
 }
 
@@ -68,4 +71,60 @@ void BaseSetting<T>::set(const std::string & str, bool append)
     }
 }
 
+template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category);
+
+template<typename T>
+void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
+{
+    args.addFlag({
+        .longName = name,
+        .description = fmt("Set the `%s` setting.", name),
+        .category = category,
+        .labels = {"value"},
+        .handler = {[this](std::string s) { overridden = true; set(s); }},
+        .experimentalFeature = experimentalFeature,
+    });
+
+    if (isAppendable())
+        args.addFlag({
+            .longName = "extra-" + name,
+            .description = fmt("Append to the `%s` setting.", name),
+            .category = category,
+            .labels = {"value"},
+            .handler = {[this](std::string s) { overridden = true; set(s, true); }},
+            .experimentalFeature = experimentalFeature,
+        });
+}
+
+#define DECLARE_CONFIG_SERIALISER(TY) \
+    template<> TY BaseSetting< TY >::parse(const std::string & str) const; \
+    template<> std::string BaseSetting< TY >::to_string() const;
+
+DECLARE_CONFIG_SERIALISER(std::string)
+DECLARE_CONFIG_SERIALISER(std::optional<std::string>)
+DECLARE_CONFIG_SERIALISER(bool)
+DECLARE_CONFIG_SERIALISER(Strings)
+DECLARE_CONFIG_SERIALISER(StringSet)
+DECLARE_CONFIG_SERIALISER(StringMap)
+DECLARE_CONFIG_SERIALISER(std::set<ExperimentalFeature>)
+
+template<typename T>
+T BaseSetting<T>::parse(const std::string & str) const
+{
+    static_assert(std::is_integral<T>::value, "Integer required.");
+
+    if (auto n = string2Int<T>(str))
+        return *n;
+    else
+        throw UsageError("setting '%s' has invalid value '%s'", name, str);
+}
+
+template<typename T>
+std::string BaseSetting<T>::to_string() const
+{
+    static_assert(std::is_integral<T>::value, "Integer required.");
+
+    return std::to_string(value);
+}
+
 }
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 085a884dc..38d406e8a 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -219,29 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
 {
 }
 
-template<typename T>
-void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
-{
-    args.addFlag({
-        .longName = name,
-        .description = fmt("Set the `%s` setting.", name),
-        .category = category,
-        .labels = {"value"},
-        .handler = {[this](std::string s) { overridden = true; set(s); }},
-        .experimentalFeature = experimentalFeature,
-    });
-
-    if (isAppendable())
-        args.addFlag({
-            .longName = "extra-" + name,
-            .description = fmt("Append to the `%s` setting.", name),
-            .category = category,
-            .labels = {"value"},
-            .handler = {[this](std::string s) { overridden = true; set(s, true); }},
-            .experimentalFeature = experimentalFeature,
-        });
-}
-
 template<> std::string BaseSetting<std::string>::parse(const std::string & str) const
 {
     return str;
@@ -252,21 +229,17 @@ template<> std::string BaseSetting<std::string>::to_string() const
     return value;
 }
 
-template<typename T>
-T BaseSetting<T>::parse(const std::string & str) const
+template<> std::optional<std::string> BaseSetting<std::optional<std::string>>::parse(const std::string & str) const
 {
-    static_assert(std::is_integral<T>::value, "Integer required.");
-    if (auto n = string2Int<T>(str))
-        return *n;
+    if (str == "")
+        return std::nullopt;
     else
-        throw UsageError("setting '%s' has invalid value '%s'", name, str);
+        return { str };
 }
 
-template<typename T>
-std::string BaseSetting<T>::to_string() const
+template<> std::string BaseSetting<std::optional<std::string>>::to_string() const
 {
-    static_assert(std::is_integral<T>::value, "Integer required.");
-    return std::to_string(value);
+    return value ? *value : "";
 }
 
 template<> bool BaseSetting<bool>::parse(const std::string & str) const
@@ -403,15 +376,25 @@ template class BaseSetting<StringSet>;
 template class BaseSetting<StringMap>;
 template class BaseSetting<std::set<ExperimentalFeature>>;
 
+static Path parsePath(const AbstractSetting & s, const std::string & str)
+{
+    if (str == "")
+        throw UsageError("setting '%s' is a path and paths cannot be empty", s.name);
+    else
+        return canonPath(str);
+}
+
 Path PathSetting::parse(const std::string & str) const
 {
-    if (str == "") {
-        if (allowEmpty)
-            return "";
-        else
-            throw UsageError("setting '%s' cannot be empty", name);
-    } else
-        return canonPath(str);
+    return parsePath(*this, str);
+}
+
+std::optional<Path> OptionalPathSetting::parse(const std::string & str) const
+{
+    if (str == "")
+        return std::nullopt;
+    else
+        return parsePath(*this, str);
 }
 
 bool GlobalConfig::set(const std::string & name, const std::string & value)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 2675baed7..cc8532587 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -353,21 +353,20 @@ public:
 /**
  * A special setting for Paths. These are automatically canonicalised
  * (e.g. "/foo//bar/" becomes "/foo/bar").
+ *
+ * It is mandatory to specify a path; i.e. the empty string is not
+ * permitted.
  */
 class PathSetting : public BaseSetting<Path>
 {
-    bool allowEmpty;
-
 public:
 
     PathSetting(Config * options,
-        bool allowEmpty,
         const Path & def,
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
         : BaseSetting<Path>(def, true, name, description, aliases)
-        , allowEmpty(allowEmpty)
     {
         options->addSetting(this);
     }
@@ -379,6 +378,30 @@ public:
     void operator =(const Path & v) { this->assign(v); }
 };
 
+/**
+ * Like `PathSetting`, but the absence of a path is also allowed.
+ *
+ * `std::optional` is used instead of the empty string for clarity.
+ */
+class OptionalPathSetting : public BaseSetting<std::optional<Path>>
+{
+public:
+
+    OptionalPathSetting(Config * options,
+        const std::optional<Path> & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<std::optional<Path>>(def, true, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    std::optional<Path> parse(const std::string & str) const override;
+
+    void operator =(const std::optional<Path> & v) { this->assign(v); }
+};
+
 struct GlobalConfig : public AbstractConfig
 {
     typedef std::vector<Config*> ConfigRegistrations;

From 6b06e97bde3d2d87a750210d46086624dc96a0f8 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Thu, 15 Jun 2023 17:12:03 -0700
Subject: [PATCH 113/183] src/libexpr/eval.hh: add link for allowed-uris option

This commit adds a link to the documentation for `--option
allowed-uris` where that option is mentioned while describing
`restrict-eval`.
---
 src/libexpr/eval.hh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index ea1f5975b..8e41bdbd0 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -741,7 +741,8 @@ struct EvalSettings : Config
           If set to `true`, the Nix evaluator will not allow access to any
           files outside of the Nix search path (as set via the `NIX_PATH`
           environment variable or the `-I` option), or to URIs outside of
-          `allowed-uris`. The default is `false`.
+          [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
+          The default is `false`.
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",

From 7bf17f8825b7aacde8b4e3c5e035f6d442d649c4 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 19 Jun 2023 05:45:08 +0200
Subject: [PATCH 114/183] Add description for file system objects (#8500)

While this is not actually a notion in the implementation, it is
explicitly described in the thesis and quite important for understanding
how the store works.

Co-authored-by: John Ericson <git@JohnEricson.me>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/SUMMARY.md.in                  |  3 +-
 .../src/architecture/file-system-object.md    | 64 +++++++++++++++++++
 doc/manual/src/glossary.md                    | 15 +++--
 3 files changed, 76 insertions(+), 6 deletions(-)
 create mode 100644 doc/manual/src/architecture/file-system-object.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index cfbb38be8..13d2e4d15 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -97,7 +97,8 @@
       - [manifest.json](command-ref/files/manifest.json.md)
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
-- [Architecture](architecture/architecture.md)
+- [Architecture and Design](architecture/architecture.md)
+  - [File System Object](architecture/file-system-object.md)
 - [Protocols](protocols/protocols.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
 - [Glossary](glossary.md)
diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/architecture/file-system-object.md
new file mode 100644
index 000000000..42f047260
--- /dev/null
+++ b/doc/manual/src/architecture/file-system-object.md
@@ -0,0 +1,64 @@
+# File System Object
+
+Nix uses a simplified model of the file system, which consists of file system objects.
+Every file system object is one of the following:
+
+ - File
+
+   - A possibly empty sequence of bytes for contents
+   - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
+
+ - Directory
+
+   Mapping of names to child file system objects
+
+ - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
+
+   An arbitrary string.
+   Nix does not assign any semantics to symbolic links.
+
+File system objects and their children form a tree.
+A bare file or symlink can be a root file system object.
+
+Nix does not encode any other file system notions such as [hard links](https://en.m.wikipedia.org/wiki/Hard_link), [permissions](https://en.m.wikipedia.org/wiki/File-system_permissions), timestamps, or other metadata.
+
+## Examples of file system objects
+
+A plain file:
+
+```
+50 B, executable: false
+```
+
+An executable file:
+
+```
+122 KB, executable: true
+```
+
+A symlink:
+
+```
+-> /usr/bin/sh
+```
+
+A directory with contents:
+
+```
+├── bin
+│   └── hello: 35 KB, executable: true
+└── share
+    ├── info
+    │   └── hello.info: 36 KB, executable: false
+    └── man
+        └── man1
+            └── hello.1.gz: 790 B, executable: false
+```
+
+A directory that contains a symlink and other directories:
+
+```
+├── bin -> share/go/bin
+├── nix-support/
+└── share/
+```
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 47a484826..ac0bb3c2f 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -85,12 +85,17 @@
 
     [store path]: #gloss-store-path
 
+  - [file system object]{#gloss-store-object}\
+    The Nix data model for representing simplified file system data.
+
+    See [File System Object](@docroot@/architecture/file-system-object.md) for details.
+
+    [file system object]: #gloss-file-system-object
+
   - [store object]{#gloss-store-object}\
-    A file that is an immediate child of the Nix store directory. These
-    can be regular files, but also entire directory trees. Store objects
-    can be sources (objects copied from outside of the store),
-    derivation outputs (objects produced by running a build task), or
-    derivations (files describing a build task).
+
+    A store object consists of a [file system object], [reference]s to other store objects, and other metadata.
+    It can be referred to by a [store path].
 
     [store object]: #gloss-store-object
 

From c404623a1d39431cf7b4ccd0b0b396a821a6eade Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 19 Jun 2023 00:04:59 -0400
Subject: [PATCH 115/183] Clean up a few things related to profiles (#8526)

- Greatly expand API docs

- Clean up code in misc ways

  - Instead of a complicated single loop on generations, do different
    operations in successive subsequent steps.

  - Avoid `ref` in one place where `&` is fine

  - Just return path instead of mutating an argument in `makeName`

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libcmd/command.cc                         |   4 +-
 src/libstore/profiles.cc                      | 123 +++++++++------
 src/libstore/profiles.hh                      | 143 +++++++++++++++++-
 .../nix-collect-garbage.cc                    |   7 +-
 src/nix-env/nix-env.cc                        |   7 +-
 src/nix-env/user-env.cc                       |   2 +-
 src/nix/profile.cc                            |   7 +-
 7 files changed, 224 insertions(+), 69 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 6c4648b34..4fc197956 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -239,9 +239,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
     if (!store) throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
     switchLink(profile2,
-        createGeneration(
-            ref<LocalFSStore>(store),
-            profile2, storePath));
+        createGeneration(*store, profile2, storePath));
 }
 
 void MixProfile::updateProfile(const BuiltPaths & buildables)
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index ba5c8583f..4e9955948 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -13,8 +13,10 @@
 namespace nix {
 
 
-/* Parse a generation name of the format
-   `<profilename>-<number>-link'. */
+/**
+ * Parse a generation name of the format
+ * `<profilename>-<number>-link'.
+ */
 static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name)
 {
     if (name.substr(0, profileName.size() + 1) != profileName + "-") return {};
@@ -28,7 +30,6 @@ static std::optional<GenerationNumber> parseName(const std::string & profileName
 }
 
 
-
 std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
 {
     Generations gens;
@@ -61,15 +62,16 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
 }
 
 
-static void makeName(const Path & profile, GenerationNumber num,
-    Path & outLink)
+/**
+ * Create a generation name that can be parsed by `parseName()`.
+ */
+static Path makeName(const Path & profile, GenerationNumber num)
 {
-    Path prefix = fmt("%1%-%2%", profile, num);
-    outLink = prefix + "-link";
+    return fmt("%s-%s-link", profile, num);
 }
 
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
+Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath)
 {
     /* The new generation number should be higher than old the
        previous ones. */
@@ -79,7 +81,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
     if (gens.size() > 0) {
         Generation last = gens.back();
 
-        if (readLink(last.path) == store->printStorePath(outPath)) {
+        if (readLink(last.path) == store.printStorePath(outPath)) {
             /* We only create a new generation symlink if it differs
                from the last one.
 
@@ -89,7 +91,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
             return last.path;
         }
 
-        num = gens.back().number;
+        num = last.number;
     } else {
         num = 0;
     }
@@ -100,9 +102,8 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
        to the permanent roots (of which the GC would have a stale
        view).  If we didn't do it this way, the GC might remove the
        user environment etc. we've just built. */
-    Path generation;
-    makeName(profile, num + 1, generation);
-    store->addPermRoot(outPath, generation);
+    Path generation = makeName(profile, num + 1);
+    store.addPermRoot(outPath, generation);
 
     return generation;
 }
@@ -117,12 +118,19 @@ static void removeFile(const Path & path)
 
 void deleteGeneration(const Path & profile, GenerationNumber gen)
 {
-    Path generation;
-    makeName(profile, gen, generation);
+    Path generation = makeName(profile, gen);
     removeFile(generation);
 }
 
-
+/**
+ * Delete a generation with dry-run mode.
+ *
+ * Like `deleteGeneration()` but:
+ *
+ *  - We log what we are going to do.
+ *
+ *  - We only actually delete if `dryRun` is false.
+ */
 static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
 {
     if (dryRun)
@@ -150,27 +158,36 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
     }
 }
 
+/**
+ * Advanced the iterator until the given predicate `cond` returns `true`.
+ */
+static inline void iterDropUntil(Generations & gens, auto && i, auto && cond)
+{
+    for (; i != gens.rend() && !cond(*i); ++i);
+}
+
 void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
 {
+    if (max == 0)
+        throw Error("Must keep at least one generation, otherwise the current one would be deleted");
+
     PathLocks lock;
     lockProfile(lock, profile);
 
-    bool fromCurGen = false;
-    auto [gens, curGen] = findGenerations(profile);
-    for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
-        if (i->number == curGen) {
-            fromCurGen = true;
-            max--;
-            continue;
-        }
-        if (fromCurGen) {
-            if (max) {
-                max--;
-                continue;
-            }
-            deleteGeneration2(profile, i->number, dryRun);
-        }
-    }
+    auto [gens, _curGen] = findGenerations(profile);
+    auto curGen = _curGen;
+
+    auto i = gens.rbegin();
+
+    // Find the current generation
+    iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; });
+
+    // Skip over `max` generations, preserving them
+    for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep);
+
+    // Delete the rest
+    for (; i != gens.rend(); ++i)
+        deleteGeneration2(profile, i->number, dryRun);
 }
 
 void deleteOldGenerations(const Path & profile, bool dryRun)
@@ -193,23 +210,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
 
     auto [gens, curGen] = findGenerations(profile);
 
-    bool canDelete = false;
-    for (auto i = gens.rbegin(); i != gens.rend(); ++i)
-        if (canDelete) {
-            assert(i->creationTime < t);
-            if (i->number != curGen)
-                deleteGeneration2(profile, i->number, dryRun);
-        } else if (i->creationTime < t) {
-            /* We may now start deleting generations, but we don't
-               delete this generation yet, because this generation was
-               still the one that was active at the requested point in
-               time. */
-            canDelete = true;
-        }
+    auto i = gens.rbegin();
+
+    // Predicate that the generation is older than the given time.
+    auto older = [&](auto & g) { return g.creationTime < t; };
+
+    // Find the first older generation, if one exists
+    iterDropUntil(gens, i, older);
+
+    /* Take the previous generation
+
+       We don't want delete this one yet because it
+       existed at the requested point in time, and
+       we want to be able to roll back to it. */
+    if (i != gens.rend()) ++i;
+
+    // Delete all previous generations (unless current).
+    for (; i != gens.rend(); ++i) {
+        /* Creating date and generations should be monotonic, so lower
+           numbered derivations should also be older. */
+        assert(older(*i));
+        if (i->number != curGen)
+            deleteGeneration2(profile, i->number, dryRun);
+    }
 }
 
 
-void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun)
+time_t parseOlderThanTimeSpec(std::string_view timeSpec)
 {
     if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
         throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
@@ -221,9 +248,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec,
     if (!days || *days < 1)
         throw UsageError("invalid number of days specifier '%1%'", timeSpec);
 
-    time_t oldTime = curTime - *days * 24 * 3600;
-
-    deleteGenerationsOlderThan(profile, oldTime, dryRun);
+    return curTime - *days * 24 * 3600;
 }
 
 
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 4e1f42e83..193c0bf21 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -1,7 +1,11 @@
 #pragma once
-///@file
+/**
+ * @file Implementation of Profiles.
+ *
+ * See the manual for additional information.
+ */
 
- #include "types.hh"
+#include "types.hh"
 #include "pathlocks.hh"
 
 #include <time.h>
@@ -12,41 +16,166 @@ namespace nix {
 class StorePath;
 
 
+/**
+ * A positive number identifying a generation for a given profile.
+ *
+ * Generation numbers are assigned sequentially. Each new generation is
+ * assigned 1 + the current highest generation number.
+ */
 typedef uint64_t GenerationNumber;
 
+/**
+ * A generation is a revision of a profile.
+ *
+ * Each generation is a mapping (key-value pair) from an identifier
+ * (`number`) to a store object (specified by `path`).
+ */
 struct Generation
 {
+    /**
+     * The number of a generation is its unique identifier within the
+     * profile.
+     */
     GenerationNumber number;
+    /**
+     * The store path identifies the store object that is the contents
+     * of the generation.
+     *
+     * These store paths / objects are not unique to the generation
+     * within a profile. Nix tries to ensure successive generations have
+     * distinct contents to avoid bloat, but nothing stops two
+     * non-adjacent generations from having the same contents.
+     *
+     * @todo Use `StorePath` instead of `Path`?
+     */
     Path path;
+
+    /**
+     * When the generation was created. This is extra metadata about the
+     * generation used to make garbage collecting old generations more
+     * convenient.
+     */
     time_t creationTime;
 };
 
+/**
+ * All the generations of a profile
+ */
 typedef std::list<Generation> Generations;
 
 
 /**
- * Returns the list of currently present generations for the specified
- * profile, sorted by generation number. Also returns the number of
- * the current generation.
+ * Find all generations for the given profile.
+ *
+ * @param profile A profile specified by its name and location combined
+ * into a path. E.g. if "foo" is the name of the profile, and "/bar/baz"
+ * is the directory it is in, then the path "/bar/baz/foo" would be the
+ * argument for this parameter.
+ *
+ * @return The pair of:
+ *
+ *   - The list of currently present generations for the specified profile,
+ *     sorted by ascending generation number.
+ *
+ *   - The number of the current/active generation.
+ *
+ * Note that the current/active generation need not be the latest one.
  */
 std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
 
 class LocalFSStore;
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
+/**
+ * Create a new generation of the given profile
+ *
+ * If the previous generation (not the currently active one!) has a
+ * distinct store object, a fresh generation number is mapped to the
+ * given store object, referenced by path. Otherwise, the previous
+ * generation is assumed.
+ *
+ * The behavior of reusing existing generations like this makes this
+ * procedure idempotent. It also avoids clutter.
+ */
+Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath);
 
+/**
+ * Unconditionally delete a generation
+ *
+ * @param profile A profile specified by its name and location combined into a path.
+ *
+ * @param gen The generation number specifying exactly which generation
+ * to delete.
+ *
+ * Because there is no check of whether the generation to delete is
+ * active, this is somewhat unsafe.
+ *
+ * @todo Should we expose this at all?
+ */
 void deleteGeneration(const Path & profile, GenerationNumber gen);
 
+/**
+ * Delete the given set of generations.
+ *
+ * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
+ *
+ * @param gensToDelete The generations to delete, specified by a set of
+ * numbers.
+ *
+ * @param dryRun Log what would be deleted instead of actually doing
+ * so.
+ *
+ * Trying to delete the currently active generation will fail, and cause
+ * no generations to be deleted.
+ */
 void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
 
+/**
+ * Delete generations older than `max` passed the current generation.
+ *
+ * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
+ *
+ * @param max How many generations to keep up to the current one. Must
+ * be at least 1 so we don't delete the current one.
+ *
+ * @param dryRun Log what would be deleted instead of actually doing
+ * so.
+ */
 void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
 
+/**
+ * Delete all generations other than the current one
+ *
+ * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
+ *
+ * @param dryRun Log what would be deleted instead of actually doing
+ * so.
+ */
 void deleteOldGenerations(const Path & profile, bool dryRun);
 
+/**
+ * Delete generations older than `t`, except for the most recent one
+ * older than `t`.
+ *
+ * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
+ *
+ * @param dryRun Log what would be deleted instead of actually doing
+ * so.
+ */
 void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
 
-void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun);
+/**
+ * Parse a temp spec intended for `deleteGenerationsOlderThan()`.
+ *
+ * Throws an exception if `timeSpec` fails to parse.
+ */
+time_t parseOlderThanTimeSpec(std::string_view timeSpec);
 
+/**
+ * Smaller wrapper around `replaceSymlink` for replacing the current
+ * generation of a profile. Does not enforce proper structure.
+ *
+ * @todo Always use `switchGeneration()` instead, and delete this.
+ */
 void switchLink(Path link, Path target);
 
 /**
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index cb1f42e35..70af53b28 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -41,9 +41,10 @@ void removeOldGenerations(std::string dir)
             }
             if (link.find("link") != std::string::npos) {
                 printInfo("removing old generations of profile %s", path);
-                if (deleteOlderThan != "")
-                    deleteGenerationsOlderThan(path, deleteOlderThan, dryRun);
-                else
+                if (deleteOlderThan != "") {
+                    auto t = parseOlderThanTimeSpec(deleteOlderThan);
+                    deleteGenerationsOlderThan(path, t, dryRun);
+                } else
                     deleteOldGenerations(path, dryRun);
             }
         } else if (type == DT_DIR) {
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 5e94f2d14..91b073b49 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -772,7 +772,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
 
     debug("switching to new user environment");
     Path generation = createGeneration(
-        ref<LocalFSStore>(store2),
+        *store2,
         globals.profile,
         drv.queryOutPath());
     switchLink(globals.profile, generation);
@@ -1356,13 +1356,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
     if (opArgs.size() == 1 && opArgs.front() == "old") {
         deleteOldGenerations(globals.profile, globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('d') != std::string::npos) {
-        deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
+        auto t = parseOlderThanTimeSpec(opArgs.front());
+        deleteGenerationsOlderThan(globals.profile, t, globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('+') != std::string::npos) {
         if (opArgs.front().size() < 2)
             throw Error("invalid number of generations '%1%'", opArgs.front());
         auto str_max = opArgs.front().substr(1);
         auto max = string2Int<GenerationNumber>(str_max);
-        if (!max || *max == 0)
+        if (!max)
             throw Error("invalid number of generations to keep '%1%'", opArgs.front());
         deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun);
     } else {
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 9e916abc4..d12d70f33 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -158,7 +158,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         }
 
         debug("switching to new user environment");
-        Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut);
+        Path generation = createGeneration(*store2, profile, topLevelOut);
         switchLink(profile, generation);
     }
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 7cea616d2..f3b73f10d 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -806,9 +806,10 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu
 
     void run(ref<Store> store) override
     {
-        if (minAge)
-            deleteGenerationsOlderThan(*profile, *minAge, dryRun);
-        else
+        if (minAge) {
+            auto t = parseOlderThanTimeSpec(*minAge);
+            deleteGenerationsOlderThan(*profile, t, dryRun);
+        } else
             deleteOldGenerations(*profile, dryRun);
     }
 };

From 966e5dc991e849d2cd26d19266ae3f065ec69c96 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 19 Jun 2023 10:39:19 +0200
Subject: [PATCH 116/183] CONTRIBUTING.md: add link to "good first issues"

---
 CONTRIBUTING.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 57a949906..4a72a8eac 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,7 +5,6 @@ We appreciate your support.
 
 Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved.
 
-
 ## Report a bug
 
 1. Check on the [GitHub issue tracker](https://github.com/NixOS/nix/issues) if your bug was already reported.
@@ -30,6 +29,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
    You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics.
 
 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue.
+
+   Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly.
    Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review.
 
 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests.

From b6e74ea5a82e5c43d42bf8386236947408748a96 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 19 Jun 2023 10:55:34 +0200
Subject: [PATCH 117/183] maintainers: add note on marking PRs as draft

as discussed with maintainers team
---
 maintainers/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/maintainers/README.md b/maintainers/README.md
index d13349438..0d520cb0c 100644
--- a/maintainers/README.md
+++ b/maintainers/README.md
@@ -117,6 +117,7 @@ Pull requests in this column are reviewed together during work meetings.
 This is both for spreading implementation knowledge and for establishing common values in code reviews.
 
 When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member.
+If significant changes are requested or reviewers cannot come to a conclusion in reasonable time, the pull request is [marked as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#converting-a-pull-request-to-a-draft).
 
 ### Assigned
 

From 469d06f9bc9b2543f48d8e633e47f9344fba35ab Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 8 Mar 2022 21:53:26 +0000
Subject: [PATCH 118/183] Split out worker protocol template definitions from
 declarations

This is generally a fine practice: Putting implementations in headers
makes them harder to read and slows compilation. Unfortunately it is
necessary for templates, but we can ameliorate that by putting them in a
separate header. Only files which need to instantiate those templates
will need to include the header with the implementation; the rest can
just include the declaration.

This is now documenting in the contributing guide.

Also, it just happens that these polymorphic serializers are the
protocol agnostic ones. (Worker and serve protocol have the same logic
for these container types.) This means by doing this general template
cleanup, we are also getting a head start on better indicating which
code is protocol-specific and which code is shared between protocols.
---
 doc/manual/src/SUMMARY.md.in                |  1 +
 doc/manual/src/contributing/cxx.md          | 28 ++++++++
 src/libstore/build/derivation-goal.cc       |  1 +
 src/libstore/build/local-derivation-goal.cc |  1 +
 src/libstore/daemon.cc                      |  1 +
 src/libstore/derivations.cc                 |  1 +
 src/libstore/export-import.cc               |  1 +
 src/libstore/legacy-ssh-store.cc            |  1 +
 src/libstore/path-info.cc                   |  1 +
 src/libstore/remote-store.cc                |  1 +
 src/libstore/worker-protocol-impl.hh        | 78 +++++++++++++++++++++
 src/libstore/worker-protocol.cc             |  1 +
 src/libstore/worker-protocol.hh             | 63 -----------------
 src/libutil/config-impl.hh                  |  3 +
 src/nix-store/nix-store.cc                  |  1 +
 15 files changed, 120 insertions(+), 63 deletions(-)
 create mode 100644 doc/manual/src/contributing/cxx.md
 create mode 100644 src/libstore/worker-protocol-impl.hh

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 13d2e4d15..bbb603713 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -106,6 +106,7 @@
   - [Hacking](contributing/hacking.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
+  - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)
diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/contributing/cxx.md
new file mode 100644
index 000000000..ff9297158
--- /dev/null
+++ b/doc/manual/src/contributing/cxx.md
@@ -0,0 +1,28 @@
+# C++ style guide
+
+Some miscellaneous notes on how we write C++.
+Formatting we hope to eventually normalize automatically, so this section is free to just discuss higher-level concerns.
+
+## The `*-impl.hh` pattern
+
+Let's start with some background info first.
+Headers, are supposed to contain declarations, not definitions.
+This allows us to change a definition without changing the declaration, and have a very small rebuild during development.
+Templates, however, need to be specialized to use-sites.
+Absent fancier techniques, templates require that the definition, not just mere declaration, must be available at use-sites in order to make that specialization on the fly as part of compiling those use-sites.
+Making definitions available like that means putting them in headers, but that is unfortunately means we get all the extra rebuilds we want to avoid by just putting declarations there as described above.
+
+The `*-impl.hh` pattern is a ham-fisted partial solution to this problem.
+It constitutes:
+
+- Declaring items only in the main `foo.hh`, including templates.
+
+- Putting template definitions in a companion `foo-impl.hh` header.
+
+Most C++ developers would accompany this by having `foo.hh` include `foo-impl.hh`, to ensure any file getting the template declarations also got the template definitions.
+But we've found not doing this has some benefits and fewer than imagined downsides.
+The fact remains that headers are rarely as minimal as they could be;
+there is often code that needs declarations from the headers but not the templates within them.
+With our pattern where `foo.hh` doesn't include `foo-impl.hh`, that means they can just include `foo.hh`
+Code that needs both just includes `foo.hh` and `foo-impl.hh`.
+This does make linking error possible where something forgets to include `foo-impl.hh` that needs it, but those are build-time only as easy to fix.
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index df7d21e54..e02000c8b 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -9,6 +9,7 @@
 #include "archive.hh"
 #include "compression.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "local-store.hh" // TODO remove, along with remaining downcasts
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 7f87cdf55..6e06f6061 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -11,6 +11,7 @@
 #include "compression.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "json-utils.hh"
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index b6dd83684..54b089b30 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -1,6 +1,7 @@
 #include "daemon.hh"
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "build-result.hh"
 #include "store-api.hh"
 #include "store-cast.hh"
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index ccb165d68..4a1f1e99f 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -5,6 +5,7 @@
 #include "util.hh"
 #include "split.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 5ea263a86..72ad77124 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 
 #include <algorithm>
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 2b7bebe9d..bff024f96 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -7,6 +7,7 @@
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "ssh.hh"
 #include "derivations.hh"
 #include "callback.hh"
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 97b72faa3..6b93976fa 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -1,5 +1,6 @@
 #include "path-info.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "store-api.hh"
 
 namespace nix {
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index c3dfb5979..f02342d32 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -6,6 +6,7 @@
 #include "build-result.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "archive.hh"
 #include "globals.hh"
 #include "derivations.hh"
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
new file mode 100644
index 000000000..509a8a5d0
--- /dev/null
+++ b/src/libstore/worker-protocol-impl.hh
@@ -0,0 +1,78 @@
+#pragma once
+/**
+ * @file
+ *
+ * Template implementations (as opposed to mere declarations).
+ *
+ * This file is an exmample of the "impl.hh" pattern. See the
+ * contributing guide.
+ */
+
+#include "worker-protocol.hh"
+
+namespace nix {
+
+template<typename T>
+std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
+{
+    std::vector<T> resSet;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        resSet.push_back(WorkerProto<T>::read(store, from));
+    }
+    return resSet;
+}
+
+template<typename T>
+void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
+{
+    out << resSet.size();
+    for (auto & key : resSet) {
+        WorkerProto<T>::write(store, out, key);
+    }
+}
+
+template<typename T>
+std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
+{
+    std::set<T> resSet;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        resSet.insert(WorkerProto<T>::read(store, from));
+    }
+    return resSet;
+}
+
+template<typename T>
+void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
+{
+    out << resSet.size();
+    for (auto & key : resSet) {
+        WorkerProto<T>::write(store, out, key);
+    }
+}
+
+template<typename K, typename V>
+std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
+{
+    std::map<K, V> resMap;
+    auto size = readNum<size_t>(from);
+    while (size--) {
+        auto k = WorkerProto<K>::read(store, from);
+        auto v = WorkerProto<V>::read(store, from);
+        resMap.insert_or_assign(std::move(k), std::move(v));
+    }
+    return resMap;
+}
+
+template<typename K, typename V>
+void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+{
+    out << resMap.size();
+    for (auto & i : resMap) {
+        WorkerProto<K>::write(store, out, i.first);
+        WorkerProto<V>::write(store, out, i.second);
+    }
+}
+
+}
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 51bb12026..49708b642 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -4,6 +4,7 @@
 #include "store-api.hh"
 #include "build-result.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "archive.hh"
 #include "derivations.hh"
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index f06332d17..18a4e11b2 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -173,67 +173,4 @@ MAKE_WORKER_PROTO(std::optional<StorePath>);
 template<>
 MAKE_WORKER_PROTO(std::optional<ContentAddress>);
 
-template<typename T>
-std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
-{
-    std::vector<T> resSet;
-    auto size = readNum<size_t>(from);
-    while (size--) {
-        resSet.push_back(WorkerProto<T>::read(store, from));
-    }
-    return resSet;
-}
-
-template<typename T>
-void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
-{
-    out << resSet.size();
-    for (auto & key : resSet) {
-        WorkerProto<T>::write(store, out, key);
-    }
-}
-
-template<typename T>
-std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
-{
-    std::set<T> resSet;
-    auto size = readNum<size_t>(from);
-    while (size--) {
-        resSet.insert(WorkerProto<T>::read(store, from));
-    }
-    return resSet;
-}
-
-template<typename T>
-void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
-{
-    out << resSet.size();
-    for (auto & key : resSet) {
-        WorkerProto<T>::write(store, out, key);
-    }
-}
-
-template<typename K, typename V>
-std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
-{
-    std::map<K, V> resMap;
-    auto size = readNum<size_t>(from);
-    while (size--) {
-        auto k = WorkerProto<K>::read(store, from);
-        auto v = WorkerProto<V>::read(store, from);
-        resMap.insert_or_assign(std::move(k), std::move(v));
-    }
-    return resMap;
-}
-
-template<typename K, typename V>
-void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
-{
-    out << resMap.size();
-    for (auto & i : resMap) {
-        WorkerProto<K>::write(store, out, i.first);
-        WorkerProto<V>::write(store, out, i.second);
-    }
-}
-
 }
diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh
index b6cae5ec3..d9726a907 100644
--- a/src/libutil/config-impl.hh
+++ b/src/libutil/config-impl.hh
@@ -4,6 +4,9 @@
  *
  * Template implementations (as opposed to mere declarations).
  *
+ * This file is an exmample of the "impl.hh" pattern. See the
+ * contributing guide.
+ *
  * One only needs to include this when one is declaring a
  * `BaseClass<CustomType>` setting, or as derived class of such an
  * instantiation.
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 61c189efb..94368e415 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -12,6 +12,7 @@
 #include "shared.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"

From 95eae0c002da5ef35e583b46a67818ebb95d3a1e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 May 2023 11:07:25 -0400
Subject: [PATCH 119/183] Put worker protocol items inside a `WorkerProto`
 struct

See API docs on that struct for why. The pasing as as template argument
doesn't yet happen in that commit, but will instead happen in later
commit.

Also make `WorkerOp` (now `Op`) and enum struct. This led us to catch
that two operations were not handled!

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/build-remote/build-remote.cc      |   2 +-
 src/libstore/build/derivation-goal.cc |   4 +-
 src/libstore/daemon.cc                | 148 ++++++++++----------
 src/libstore/derivations.cc           |   4 +-
 src/libstore/export-import.cc         |   4 +-
 src/libstore/legacy-ssh-store.cc      |  16 +--
 src/libstore/path-info.cc             |   4 +-
 src/libstore/remote-store.cc          | 130 +++++++++---------
 src/libstore/worker-protocol-impl.hh  |  28 ++--
 src/libstore/worker-protocol.cc       |  56 ++++----
 src/libstore/worker-protocol.hh       | 186 ++++++++++++++++----------
 src/nix-store/nix-store.cc            |  18 +--
 12 files changed, 327 insertions(+), 273 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 323e04fdb..2fb17d06f 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -299,7 +299,7 @@ connected:
             !trusted || *trusted;
         });
 
-        // See the very large comment in `case wopBuildDerivation:` in
+        // See the very large comment in `case WorkerProto::Op::BuildDerivation:` in
         // `src/libstore/daemon.cc` that explains the trust model here.
         //
         // This condition mirrors that: that code enforces the "rules" outlined there;
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index e02000c8b..1a946e3d2 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1153,7 +1153,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    workerProtoWrite(worker.store, hook->sink, inputPaths);
+    WorkerProto::write(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1164,7 +1164,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        workerProtoWrite(worker.store, hook->sink, missingOutputs);
+        WorkerProto::write(worker.store, hook->sink, missingOutputs);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 54b089b30..7eba1a79d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -264,7 +264,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
 {
     std::vector<DerivedPath> reqs;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
-        reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
+        reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, from);
     } else {
         for (auto & s : readStrings<Strings>(from))
             reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
@@ -274,11 +274,11 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
 
 static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
-    Source & from, BufferedSink & to, unsigned int op)
+    Source & from, BufferedSink & to, WorkerProto::Op op)
 {
     switch (op) {
 
-    case wopIsValidPath: {
+    case WorkerProto::Op::IsValidPath: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         bool result = store->isValidPath(path);
@@ -287,8 +287,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryValidPaths: {
-        auto paths = WorkerProto<StorePathSet>::read(*store, from);
+    case WorkerProto::Op::QueryValidPaths: {
+        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
 
         SubstituteFlag substitute = NoSubstitute;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@@ -301,11 +301,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
-        workerProtoWrite(*store, to, res);
+        WorkerProto::write(*store, to, res);
         break;
     }
 
-    case wopHasSubstitutes: {
+    case WorkerProto::Op::HasSubstitutes: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths; // FIXME
@@ -316,16 +316,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQuerySubstitutablePaths: {
-        auto paths = WorkerProto<StorePathSet>::read(*store, from);
+    case WorkerProto::Op::QuerySubstitutablePaths: {
+        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        workerProtoWrite(*store, to, res);
+        WorkerProto::write(*store, to, res);
         break;
     }
 
-    case wopQueryPathHash: {
+    case WorkerProto::Op::QueryPathHash: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto hash = store->queryPathInfo(path)->narHash;
@@ -334,27 +334,27 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryReferences:
-    case wopQueryReferrers:
-    case wopQueryValidDerivers:
-    case wopQueryDerivationOutputs: {
+    case WorkerProto::Op::QueryReferences:
+    case WorkerProto::Op::QueryReferrers:
+    case WorkerProto::Op::QueryValidDerivers:
+    case WorkerProto::Op::QueryDerivationOutputs: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths;
-        if (op == wopQueryReferences)
+        if (op == WorkerProto::Op::QueryReferences)
             for (auto & i : store->queryPathInfo(path)->references)
                 paths.insert(i);
-        else if (op == wopQueryReferrers)
+        else if (op == WorkerProto::Op::QueryReferrers)
             store->queryReferrers(path, paths);
-        else if (op == wopQueryValidDerivers)
+        else if (op == WorkerProto::Op::QueryValidDerivers)
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        workerProtoWrite(*store, to, paths);
+        WorkerProto::write(*store, to, paths);
         break;
     }
 
-    case wopQueryDerivationOutputNames: {
+    case WorkerProto::Op::QueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto names = store->readDerivation(path).outputNames();
@@ -363,16 +363,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryDerivationOutputMap: {
+    case WorkerProto::Op::QueryDerivationOutputMap: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        workerProtoWrite(*store, to, outputs);
+        WorkerProto::write(*store, to, outputs);
         break;
     }
 
-    case wopQueryDeriver: {
+    case WorkerProto::Op::QueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto info = store->queryPathInfo(path);
@@ -381,7 +381,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryPathFromHashPart: {
+    case WorkerProto::Op::QueryPathFromHashPart: {
         auto hashPart = readString(from);
         logger->startWork();
         auto path = store->queryPathFromHashPart(hashPart);
@@ -390,11 +390,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddToStore: {
+    case WorkerProto::Op::AddToStore: {
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = WorkerProto<StorePathSet>::read(*store, from);
+            auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, from);
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -476,7 +476,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddMultipleToStore: {
+    case WorkerProto::Op::AddMultipleToStore: {
         bool repair, dontCheckSigs;
         from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
@@ -493,10 +493,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddTextToStore: {
+    case WorkerProto::Op::AddTextToStore: {
         std::string suffix = readString(from);
         std::string s = readString(from);
-        auto refs = WorkerProto<StorePathSet>::read(*store, from);
+        auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, from);
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -504,7 +504,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopExportPath: {
+    case WorkerProto::Op::ExportPath: {
         auto path = store->parseStorePath(readString(from));
         readInt(from); // obsolete
         logger->startWork();
@@ -515,7 +515,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopImportPaths: {
+    case WorkerProto::Op::ImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
         auto paths = store->importPaths(source,
@@ -527,7 +527,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopBuildPaths: {
+    case WorkerProto::Op::BuildPaths: {
         auto drvs = readDerivedPaths(*store, clientVersion, from);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
@@ -552,7 +552,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopBuildPathsWithResults: {
+    case WorkerProto::Op::BuildPathsWithResults: {
         auto drvs = readDerivedPaths(*store, clientVersion, from);
         BuildMode mode = bmNormal;
         mode = (BuildMode) readInt(from);
@@ -568,12 +568,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto results = store->buildPathsWithResults(drvs, mode);
         logger->stopWork();
 
-        workerProtoWrite(*store, to, results);
+        WorkerProto::write(*store, to, results);
 
         break;
     }
 
-    case wopBuildDerivation: {
+    case WorkerProto::Op::BuildDerivation: {
         auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
         readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
@@ -645,12 +645,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             DrvOutputs builtOutputs;
             for (auto & [output, realisation] : res.builtOutputs)
                 builtOutputs.insert_or_assign(realisation.id, realisation);
-            workerProtoWrite(*store, to, builtOutputs);
+            WorkerProto::write(*store, to, builtOutputs);
         }
         break;
     }
 
-    case wopEnsurePath: {
+    case WorkerProto::Op::EnsurePath: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         store->ensurePath(path);
@@ -659,7 +659,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddTempRoot: {
+    case WorkerProto::Op::AddTempRoot: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         store->addTempRoot(path);
@@ -668,7 +668,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddIndirectRoot: {
+    case WorkerProto::Op::AddIndirectRoot: {
         Path path = absPath(readString(from));
 
         logger->startWork();
@@ -681,14 +681,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     // Obsolete.
-    case wopSyncWithGC: {
+    case WorkerProto::Op::SyncWithGC: {
         logger->startWork();
         logger->stopWork();
         to << 1;
         break;
     }
 
-    case wopFindRoots: {
+    case WorkerProto::Op::FindRoots: {
         logger->startWork();
         auto & gcStore = require<GcStore>(*store);
         Roots roots = gcStore.findRoots(!trusted);
@@ -707,10 +707,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopCollectGarbage: {
+    case WorkerProto::Op::CollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
+        options.pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, from);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -731,7 +731,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopSetOptions: {
+    case WorkerProto::Op::SetOptions: {
 
         ClientSettings clientSettings;
 
@@ -768,7 +768,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQuerySubstitutablePathInfo: {
+    case WorkerProto::Op::QuerySubstitutablePathInfo: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
@@ -780,22 +780,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            workerProtoWrite(*store, to, i->second.references);
+            WorkerProto::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
         break;
     }
 
-    case wopQuerySubstitutablePathInfos: {
+    case WorkerProto::Op::QuerySubstitutablePathInfos: {
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = WorkerProto<StorePathSet>::read(*store, from);
+            auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
+            pathsMap = WorkerProto::Serialise<StorePathCAMap>::read(*store, from);
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -803,21 +803,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            workerProtoWrite(*store, to, i.second.references);
+            WorkerProto::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
     }
 
-    case wopQueryAllValidPaths: {
+    case WorkerProto::Op::QueryAllValidPaths: {
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        workerProtoWrite(*store, to, paths);
+        WorkerProto::write(*store, to, paths);
         break;
     }
 
-    case wopQueryPathInfo: {
+    case WorkerProto::Op::QueryPathInfo: {
         auto path = store->parseStorePath(readString(from));
         std::shared_ptr<const ValidPathInfo> info;
         logger->startWork();
@@ -838,14 +838,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopOptimiseStore:
+    case WorkerProto::Op::OptimiseStore:
         logger->startWork();
         store->optimiseStore();
         logger->stopWork();
         to << 1;
         break;
 
-    case wopVerifyStore: {
+    case WorkerProto::Op::VerifyStore: {
         bool checkContents, repair;
         from >> checkContents >> repair;
         logger->startWork();
@@ -857,7 +857,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddSignatures: {
+    case WorkerProto::Op::AddSignatures: {
         auto path = store->parseStorePath(readString(from));
         StringSet sigs = readStrings<StringSet>(from);
         logger->startWork();
@@ -869,7 +869,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopNarFromPath: {
+    case WorkerProto::Op::NarFromPath: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
@@ -877,7 +877,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopAddToStoreNar: {
+    case WorkerProto::Op::AddToStoreNar: {
         bool repair, dontCheckSigs;
         auto path = store->parseStorePath(readString(from));
         auto deriver = readString(from);
@@ -885,7 +885,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = WorkerProto<StorePathSet>::read(*store, from);
+        info.references = WorkerProto::Serialise<StorePathSet>::read(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = ContentAddress::parseOpt(readString(from));
@@ -929,21 +929,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
-    case wopQueryMissing: {
+    case WorkerProto::Op::QueryMissing: {
         auto targets = readDerivedPaths(*store, clientVersion, from);
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        workerProtoWrite(*store, to, willBuild);
-        workerProtoWrite(*store, to, willSubstitute);
-        workerProtoWrite(*store, to, unknown);
+        WorkerProto::write(*store, to, willBuild);
+        WorkerProto::write(*store, to, willSubstitute);
+        WorkerProto::write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
 
-    case wopRegisterDrvOutput: {
+    case WorkerProto::Op::RegisterDrvOutput: {
         logger->startWork();
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             auto outputId = DrvOutput::parse(readString(from));
@@ -951,14 +951,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             store->registerDrvOutput(Realisation{
                 .id = outputId, .outPath = outputPath});
         } else {
-            auto realisation = WorkerProto<Realisation>::read(*store, from);
+            auto realisation = WorkerProto::Serialise<Realisation>::read(*store, from);
             store->registerDrvOutput(realisation);
         }
         logger->stopWork();
         break;
     }
 
-    case wopQueryRealisation: {
+    case WorkerProto::Op::QueryRealisation: {
         logger->startWork();
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
@@ -966,16 +966,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             std::set<StorePath> outPaths;
             if (info) outPaths.insert(info->outPath);
-            workerProtoWrite(*store, to, outPaths);
+            WorkerProto::write(*store, to, outPaths);
         } else {
             std::set<Realisation> realisations;
             if (info) realisations.insert(*info);
-            workerProtoWrite(*store, to, realisations);
+            WorkerProto::write(*store, to, realisations);
         }
         break;
     }
 
-    case wopAddBuildLog: {
+    case WorkerProto::Op::AddBuildLog: {
         StorePath path{readString(from)};
         logger->startWork();
         if (!trusted)
@@ -992,6 +992,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case WorkerProto::Op::QueryFailedPaths:
+    case WorkerProto::Op::ClearFailedPaths:
+        throw Error("Removed operation %1%", op);
+
     default:
         throw Error("invalid operation %1%", op);
     }
@@ -1046,7 +1050,7 @@ void processConnection(
         auto temp = trusted
             ? store->isTrustedClient()
             : std::optional { NotTrusted };
-        workerProtoWrite(*store, to, temp);
+        WorkerProto::write(*store, to, temp);
     }
 
     /* Send startup error messages to the client. */
@@ -1059,9 +1063,9 @@ void processConnection(
 
         /* Process client requests. */
         while (true) {
-            WorkerOp op;
+            WorkerProto::Op op;
             try {
-                op = (WorkerOp) readInt(from);
+                op = (enum WorkerProto::Op) readInt(from);
             } catch (Interrupted & e) {
                 break;
             } catch (EndOfFile & e) {
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 4a1f1e99f..2295ff9e8 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -750,7 +750,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
+    drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store, in);
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -798,7 +798,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.raw());
     }
-    workerProtoWrite(store, out, drv.inputSrcs);
+    WorkerProto::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 72ad77124..b4100bace 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -46,7 +46,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    workerProtoWrite(*this, teeSink, info->references);
+    WorkerProto::write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -74,7 +74,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
 
-        auto references = WorkerProto<StorePathSet>::read(*this, source);
+        auto references = WorkerProto::Serialise<StorePathSet>::read(*this, source);
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index bff024f96..7d006f451 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -147,7 +147,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info->references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -181,7 +181,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            workerProtoWrite(*this, conn->to, info.references);
+            WorkerProto::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -210,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            workerProtoWrite(*this, conn->to, info.references);
+            WorkerProto::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -295,7 +295,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
+            auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, conn->from);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -370,10 +370,10 @@ public:
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        workerProtoWrite(*this, conn->to, paths);
+        WorkerProto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
+        for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, conn->from))
             out.insert(i);
     }
 
@@ -386,10 +386,10 @@ public:
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        workerProtoWrite(*this, conn->to, paths);
+        WorkerProto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
     }
 
     void connect() override
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 6b93976fa..9971de1dd 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -133,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
     auto narHash = Hash::parseAny(readString(source), htSHA256);
     ValidPathInfo info(path, narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    info.references = WorkerProto<StorePathSet>::read(store, source);
+    info.references = WorkerProto::Serialise<StorePathSet>::read(store, source);
     source >> info.registrationTime >> info.narSize;
     if (format >= 16) {
         source >> info.ultimate;
@@ -154,7 +154,7 @@ void ValidPathInfo::write(
         sink << store.printStorePath(path);
     sink << (deriver ? store.printStorePath(*deriver) : "")
          << narHash.to_string(Base16, false);
-    workerProtoWrite(store, sink, references);
+    WorkerProto::write(store, sink, references);
     sink << registrationTime << narSize;
     if (format >= 16) {
         sink << ultimate
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f02342d32..9253ce09e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -101,7 +101,7 @@ void RemoteStore::initConnection(Connection & conn)
         }
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
-            conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
+            conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn.from);
         } else {
             // We don't know the answer; protocol to old.
             conn.remoteTrustsUs = std::nullopt;
@@ -120,7 +120,7 @@ void RemoteStore::initConnection(Connection & conn)
 
 void RemoteStore::setOptions(Connection & conn)
 {
-    conn.to << wopSetOptions
+    conn.to << WorkerProto::Op::SetOptions
        << settings.keepFailed
        << settings.keepGoing
        << settings.tryFallback
@@ -212,7 +212,7 @@ void RemoteStore::setOptions()
 bool RemoteStore::isValidPathUncached(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopIsValidPath << printStorePath(path);
+    conn->to << WorkerProto::Op::IsValidPath << printStorePath(path);
     conn.processStderr();
     return readInt(conn->from);
 }
@@ -227,13 +227,13 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
             if (isValidPath(i)) res.insert(i);
         return res;
     } else {
-        conn->to << wopQueryValidPaths;
-        workerProtoWrite(*this, conn->to, paths);
+        conn->to << WorkerProto::Op::QueryValidPaths;
+        WorkerProto::write(*this, conn->to, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
             conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
         }
         conn.processStderr();
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -241,9 +241,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
 StorePathSet RemoteStore::queryAllValidPaths()
 {
     auto conn(getConnection());
-    conn->to << wopQueryAllValidPaths;
+    conn->to << WorkerProto::Op::QueryAllValidPaths;
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -253,16 +253,16 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         StorePathSet res;
         for (auto & i : paths) {
-            conn->to << wopHasSubstitutes << printStorePath(i);
+            conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i);
             conn.processStderr();
             if (readInt(conn->from)) res.insert(i);
         }
         return res;
     } else {
-        conn->to << wopQuerySubstitutablePaths;
-        workerProtoWrite(*this, conn->to, paths);
+        conn->to << WorkerProto::Op::QuerySubstitutablePaths;
+        WorkerProto::write(*this, conn->to, paths);
         conn.processStderr();
-        return WorkerProto<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -277,14 +277,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
 
         for (auto & i : pathsMap) {
             SubstitutablePathInfo info;
-            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
+            conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first);
             conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -292,14 +292,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
 
     } else {
 
-        conn->to << wopQuerySubstitutablePathInfos;
+        conn->to << WorkerProto::Op::QuerySubstitutablePathInfos;
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            workerProtoWrite(*this, conn->to, paths);
+            WorkerProto::write(*this, conn->to, paths);
         } else
-            workerProtoWrite(*this, conn->to, pathsMap);
+            WorkerProto::write(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -307,7 +307,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
+            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -323,7 +323,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
         std::shared_ptr<const ValidPathInfo> info;
         {
             auto conn(getConnection());
-            conn->to << wopQueryPathInfo << printStorePath(path);
+            conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path);
             try {
                 conn.processStderr();
             } catch (Error & e) {
@@ -348,9 +348,9 @@ void RemoteStore::queryReferrers(const StorePath & path,
     StorePathSet & referrers)
 {
     auto conn(getConnection());
-    conn->to << wopQueryReferrers << printStorePath(path);
+    conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
+    for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, conn->from))
         referrers.insert(i);
 }
 
@@ -358,9 +358,9 @@ void RemoteStore::queryReferrers(const StorePath & path,
 StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopQueryValidDerivers << printStorePath(path);
+    conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -370,9 +370,9 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
         return Store::queryDerivationOutputs(path);
     }
     auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputs << printStorePath(path);
+    conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return WorkerProto<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -380,9 +380,9 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
 {
     if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
         auto conn(getConnection());
-        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+        conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
         conn.processStderr();
-        return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
+        return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
     } else {
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -403,7 +403,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
-    conn->to << wopQueryPathFromHashPart << hashPart;
+    conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart;
     conn.processStderr();
     Path path = readString(conn->from);
     if (path.empty()) return {};
@@ -425,10 +425,10 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
 
         conn->to
-            << wopAddToStore
+            << WorkerProto::Op::AddToStore
             << name
             << caMethod.render(hashType);
-        workerProtoWrite(*this, conn->to, references);
+        WorkerProto::write(*this, conn->to, references);
         conn->to << repair;
 
         // The dump source may invoke the store, so we need to make some room.
@@ -452,13 +452,13 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                     throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
                         name, printHashType(hashType));
                 std::string s = dump.drain();
-                conn->to << wopAddTextToStore << name << s;
-                workerProtoWrite(*this, conn->to, references);
+                conn->to << WorkerProto::Op::AddTextToStore << name << s;
+                WorkerProto::write(*this, conn->to, references);
                 conn.processStderr();
             },
             [&](const FileIngestionMethod & fim) -> void {
                 conn->to
-                    << wopAddToStore
+                    << WorkerProto::Op::AddToStore
                     << name
                     << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
                     << (fim == FileIngestionMethod::Recursive ? 1 : 0)
@@ -510,7 +510,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
-        conn->to << wopImportPaths;
+        conn->to << WorkerProto::Op::ImportPaths;
 
         auto source2 = sinkToSource([&](Sink & sink) {
             sink << 1 // == path follows
@@ -519,7 +519,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            workerProtoWrite(*this, sink, info.references);
+            WorkerProto::write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -529,16 +529,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
+        auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
         assert(importedPaths.size() <= 1);
     }
 
     else {
-        conn->to << wopAddToStoreNar
+        conn->to << WorkerProto::Op::AddToStoreNar
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        workerProtoWrite(*this, conn->to, info.references);
+        WorkerProto::write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -582,7 +582,7 @@ void RemoteStore::addMultipleToStore(
     if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) {
         auto conn(getConnection());
         conn->to
-            << wopAddMultipleToStore
+            << WorkerProto::Op::AddMultipleToStore
             << repair
             << !checkSigs;
         conn.withFramedSink([&](Sink & sink) {
@@ -606,12 +606,12 @@ StorePath RemoteStore::addTextToStore(
 void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
-    conn->to << wopRegisterDrvOutput;
+    conn->to << WorkerProto::Op::RegisterDrvOutput;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
         conn->to << info.id.to_string();
         conn->to << std::string(info.outPath.to_string());
     } else {
-        workerProtoWrite(*this, conn->to, info);
+        WorkerProto::write(*this, conn->to, info);
     }
     conn.processStderr();
 }
@@ -627,19 +627,19 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
             return callback(nullptr);
         }
 
-        conn->to << wopQueryRealisation;
+        conn->to << WorkerProto::Op::QueryRealisation;
         conn->to << id.to_string();
         conn.processStderr();
 
         auto real = [&]() -> std::shared_ptr<const Realisation> {
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-                auto outPaths = WorkerProto<std::set<StorePath>>::read(
+                auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read(
                     *this, conn->from);
                 if (outPaths.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
             } else {
-                auto realisations = WorkerProto<std::set<Realisation>>::read(
+                auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(
                     *this, conn->from);
                 if (realisations.empty())
                     return nullptr;
@@ -654,7 +654,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
 {
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
-        workerProtoWrite(store, conn->to, reqs);
+        WorkerProto::write(store, conn->to, reqs);
     } else {
         Strings ss;
         for (auto & p : reqs) {
@@ -695,7 +695,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
     copyDrvsFromEvalStore(drvPaths, evalStore);
 
     auto conn(getConnection());
-    conn->to << wopBuildPaths;
+    conn->to << WorkerProto::Op::BuildPaths;
     assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
     writeDerivedPaths(*this, conn, drvPaths);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
@@ -720,11 +720,11 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
     auto & conn = *conn_;
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
-        conn->to << wopBuildPathsWithResults;
+        conn->to << WorkerProto::Op::BuildPathsWithResults;
         writeDerivedPaths(*this, conn, paths);
         conn->to << buildMode;
         conn.processStderr();
-        return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
+        return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, conn->from);
     } else {
         // Avoid deadlock.
         conn_.reset();
@@ -796,7 +796,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
     BuildMode buildMode)
 {
     auto conn(getConnection());
-    conn->to << wopBuildDerivation << printStorePath(drvPath);
+    conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath);
     writeDerivation(conn->to, *this, drv);
     conn->to << buildMode;
     conn.processStderr();
@@ -807,7 +807,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
         conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
-        auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
+        auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, conn->from);
         for (auto && [output, realisation] : builtOutputs)
             res.builtOutputs.insert_or_assign(
                 std::move(output.outputName),
@@ -820,7 +820,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
 void RemoteStore::ensurePath(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopEnsurePath << printStorePath(path);
+    conn->to << WorkerProto::Op::EnsurePath << printStorePath(path);
     conn.processStderr();
     readInt(conn->from);
 }
@@ -829,7 +829,7 @@ void RemoteStore::ensurePath(const StorePath & path)
 void RemoteStore::addTempRoot(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopAddTempRoot << printStorePath(path);
+    conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path);
     conn.processStderr();
     readInt(conn->from);
 }
@@ -838,7 +838,7 @@ void RemoteStore::addTempRoot(const StorePath & path)
 void RemoteStore::addIndirectRoot(const Path & path)
 {
     auto conn(getConnection());
-    conn->to << wopAddIndirectRoot << path;
+    conn->to << WorkerProto::Op::AddIndirectRoot << path;
     conn.processStderr();
     readInt(conn->from);
 }
@@ -847,7 +847,7 @@ void RemoteStore::addIndirectRoot(const Path & path)
 Roots RemoteStore::findRoots(bool censor)
 {
     auto conn(getConnection());
-    conn->to << wopFindRoots;
+    conn->to << WorkerProto::Op::FindRoots;
     conn.processStderr();
     size_t count = readNum<size_t>(conn->from);
     Roots result;
@@ -865,8 +865,8 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     auto conn(getConnection());
 
     conn->to
-        << wopCollectGarbage << options.action;
-    workerProtoWrite(*this, conn->to, options.pathsToDelete);
+        << WorkerProto::Op::CollectGarbage << options.action;
+    WorkerProto::write(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -888,7 +888,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 void RemoteStore::optimiseStore()
 {
     auto conn(getConnection());
-    conn->to << wopOptimiseStore;
+    conn->to << WorkerProto::Op::OptimiseStore;
     conn.processStderr();
     readInt(conn->from);
 }
@@ -897,7 +897,7 @@ void RemoteStore::optimiseStore()
 bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 {
     auto conn(getConnection());
-    conn->to << wopVerifyStore << checkContents << repair;
+    conn->to << WorkerProto::Op::VerifyStore << checkContents << repair;
     conn.processStderr();
     return readInt(conn->from);
 }
@@ -906,7 +906,7 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     auto conn(getConnection());
-    conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
+    conn->to << WorkerProto::Op::AddSignatures << printStorePath(storePath) << sigs;
     conn.processStderr();
     readInt(conn->from);
 }
@@ -922,12 +922,12 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
             // Don't hold the connection handle in the fallback case
             // to prevent a deadlock.
             goto fallback;
-        conn->to << wopQueryMissing;
+        conn->to << WorkerProto::Op::QueryMissing;
         writeDerivedPaths(*this, conn, targets);
         conn.processStderr();
-        willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
-        willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
-        unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
+        willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        unknown = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
         conn->from >> downloadSize >> narSize;
         return;
     }
@@ -941,7 +941,7 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
 void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 {
     auto conn(getConnection());
-    conn->to << wopAddBuildLog << drvPath.to_string();
+    conn->to << WorkerProto::Op::AddBuildLog << drvPath.to_string();
     StringSource source(log);
     conn.withFramedSink([&](Sink & sink) {
         source.drainInto(sink);
@@ -993,7 +993,7 @@ RemoteStore::Connection::~Connection()
 void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
 {
     auto conn(connections->get());
-    conn->to << wopNarFromPath << printStorePath(path);
+    conn->to << WorkerProto::Op::NarFromPath << printStorePath(path);
     conn->processStderr();
     copyNAR(conn->from, sink);
 }
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
index 509a8a5d0..f34e55c54 100644
--- a/src/libstore/worker-protocol-impl.hh
+++ b/src/libstore/worker-protocol-impl.hh
@@ -13,65 +13,65 @@
 namespace nix {
 
 template<typename T>
-std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
+std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, Source & from)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.push_back(WorkerProto<T>::read(store, from));
+        resSet.push_back(WorkerProto::Serialise<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
+void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        WorkerProto<T>::write(store, out, key);
+        WorkerProto::Serialise<T>::write(store, out, key);
     }
 }
 
 template<typename T>
-std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
+std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, Source & from)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.insert(WorkerProto<T>::read(store, from));
+        resSet.insert(WorkerProto::Serialise<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
+void WorkerProto::Serialise<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        WorkerProto<T>::write(store, out, key);
+        WorkerProto::Serialise<T>::write(store, out, key);
     }
 }
 
 template<typename K, typename V>
-std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
+std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, Source & from)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(from);
     while (size--) {
-        auto k = WorkerProto<K>::read(store, from);
-        auto v = WorkerProto<V>::read(store, from);
+        auto k = WorkerProto::Serialise<K>::read(store, from);
+        auto v = WorkerProto::Serialise<V>::read(store, from);
         resMap.insert_or_assign(std::move(k), std::move(v));
     }
     return resMap;
 }
 
 template<typename K, typename V>
-void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
 {
     out << resMap.size();
     for (auto & i : resMap) {
-        WorkerProto<K>::write(store, out, i.first);
-        WorkerProto<V>::write(store, out, i.second);
+        WorkerProto::Serialise<K>::write(store, out, i.first);
+        WorkerProto::Serialise<V>::write(store, out, i.second);
     }
 }
 
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 49708b642..0bf9e4d68 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -12,29 +12,29 @@
 
 namespace nix {
 
-std::string WorkerProto<std::string>::read(const Store & store, Source & from)
+std::string WorkerProto::Serialise<std::string>::read(const Store & store, Source & from)
 {
     return readString(from);
 }
 
-void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
+void WorkerProto::Serialise<std::string>::write(const Store & store, Sink & out, const std::string & str)
 {
     out << str;
 }
 
 
-StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
+StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, Source & from)
 {
     return store.parseStorePath(readString(from));
 }
 
-void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
+void WorkerProto::Serialise<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
 {
     out << store.printStorePath(storePath);
 }
 
 
-std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
+std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
 {
     auto temp = readNum<uint8_t>(from);
     switch (temp) {
@@ -49,7 +49,7 @@ std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const S
     }
 }
 
-void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
         out << (uint8_t)0;
@@ -68,30 +68,30 @@ void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink &
 }
 
 
-ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
+ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, Source & from)
 {
     return ContentAddress::parse(readString(from));
 }
 
-void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
+void WorkerProto::Serialise<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
 {
     out << renderContentAddress(ca);
 }
 
 
-DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
+DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Source & from)
 {
     auto s = readString(from);
     return DerivedPath::parseLegacy(store, s);
 }
 
-void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
+void WorkerProto::Serialise<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
 {
     out << req.to_string_legacy(store);
 }
 
 
-Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
+Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, Source & from)
 {
     std::string rawInput = readString(from);
     return Realisation::fromJSON(
@@ -100,41 +100,41 @@ Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
     );
 }
 
-void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
+void WorkerProto::Serialise<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
 {
     out << realisation.toJSON().dump();
 }
 
 
-DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
+DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, Source & from)
 {
     return DrvOutput::parse(readString(from));
 }
 
-void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+void WorkerProto::Serialise<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
 {
     out << drvOutput.to_string();
 }
 
 
-KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
+KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, Source & from)
 {
-    auto path = WorkerProto<DerivedPath>::read(store, from);
-    auto br = WorkerProto<BuildResult>::read(store, from);
+    auto path = WorkerProto::Serialise<DerivedPath>::read(store, from);
+    auto br = WorkerProto::Serialise<BuildResult>::read(store, from);
     return KeyedBuildResult {
         std::move(br),
         /* .path = */ std::move(path),
     };
 }
 
-void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
+void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
 {
-    workerProtoWrite(store, to, res.path);
-    workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
+    WorkerProto::write(store, to, res.path);
+    WorkerProto::write(store, to, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
+BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Source & from)
 {
     BuildResult res;
     res.status = (BuildResult::Status) readInt(from);
@@ -144,7 +144,7 @@ BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
         >> res.isNonDeterministic
         >> res.startTime
         >> res.stopTime;
-    auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
+    auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, from);
     for (auto && [output, realisation] : builtOutputs)
         res.builtOutputs.insert_or_assign(
             std::move(output.outputName),
@@ -152,7 +152,7 @@ BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
     return res;
 }
 
-void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
+void WorkerProto::Serialise<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
 {
     to
         << res.status
@@ -164,28 +164,28 @@ void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const Build
     DrvOutputs builtOutputs;
     for (auto & [output, realisation] : res.builtOutputs)
         builtOutputs.insert_or_assign(realisation.id, realisation);
-    workerProtoWrite(store, to, builtOutputs);
+    WorkerProto::write(store, to, builtOutputs);
 }
 
 
-std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
+std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, Source & from)
 {
     auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
 {
     out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
+std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, Source & from)
 {
     return ContentAddress::parseOpt(readString(from));
 }
 
-void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
 {
     out << (caOpt ? renderContentAddress(*caOpt) : "");
 }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 18a4e11b2..cd6801290 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -14,57 +14,6 @@ namespace nix {
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
-/**
- * Enumeration of all the request types for the "worker protocol", used
- * by unix:// and ssh-ng:// stores.
- */
-typedef enum {
-    wopIsValidPath = 1,
-    wopHasSubstitutes = 3,
-    wopQueryPathHash = 4, // obsolete
-    wopQueryReferences = 5, // obsolete
-    wopQueryReferrers = 6,
-    wopAddToStore = 7,
-    wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore
-    wopBuildPaths = 9,
-    wopEnsurePath = 10,
-    wopAddTempRoot = 11,
-    wopAddIndirectRoot = 12,
-    wopSyncWithGC = 13,
-    wopFindRoots = 14,
-    wopExportPath = 16, // obsolete
-    wopQueryDeriver = 18, // obsolete
-    wopSetOptions = 19,
-    wopCollectGarbage = 20,
-    wopQuerySubstitutablePathInfo = 21,
-    wopQueryDerivationOutputs = 22, // obsolete
-    wopQueryAllValidPaths = 23,
-    wopQueryFailedPaths = 24,
-    wopClearFailedPaths = 25,
-    wopQueryPathInfo = 26,
-    wopImportPaths = 27, // obsolete
-    wopQueryDerivationOutputNames = 28, // obsolete
-    wopQueryPathFromHashPart = 29,
-    wopQuerySubstitutablePathInfos = 30,
-    wopQueryValidPaths = 31,
-    wopQuerySubstitutablePaths = 32,
-    wopQueryValidDerivers = 33,
-    wopOptimiseStore = 34,
-    wopVerifyStore = 35,
-    wopBuildDerivation = 36,
-    wopAddSignatures = 37,
-    wopNarFromPath = 38,
-    wopAddToStoreNar = 39,
-    wopQueryMissing = 40,
-    wopQueryDerivationOutputMap = 41,
-    wopRegisterDrvOutput = 42,
-    wopQueryRealisation = 43,
-    wopAddMultipleToStore = 44,
-    wopAddBuildLog = 45,
-    wopBuildPathsWithResults = 46,
-} WorkerOp;
-
-
 #define STDERR_NEXT  0x6f6c6d67
 #define STDERR_READ  0x64617461 // data needed from source
 #define STDERR_WRITE 0x64617416 // data for sink
@@ -78,7 +27,7 @@ typedef enum {
 class Store;
 struct Source;
 
-// items being serialized
+// items being serialised
 struct DerivedPath;
 struct DrvOutput;
 struct Realisation;
@@ -88,31 +37,132 @@ enum TrustedFlag : bool;
 
 
 /**
- * Data type for canonical pairs of serializers for the worker protocol.
+ * The "worker protocol", used by unix:// and ssh-ng:// stores.
  *
- * See https://en.cppreference.com/w/cpp/language/adl for the broader
- * concept of what is going on here.
+ * This `struct` is basically just a `namespace`; We use a type rather
+ * than a namespace just so we can use it as a template argument.
  */
-template<typename T>
-struct WorkerProto {
-    static T read(const Store & store, Source & from);
-    static void write(const Store & store, Sink & out, const T & t);
+struct WorkerProto
+{
+    /**
+     * Enumeration of all the request types for the protocol.
+     */
+    enum struct Op : uint64_t;
+
+    /**
+     * Data type for canonical pairs of serialisers for the worker protocol.
+     *
+     * See https://en.cppreference.com/w/cpp/language/adl for the broader
+     * concept of what is going on here.
+     */
+    template<typename T>
+    struct Serialise;
+    // This is the definition of `Serialise` we *want* to put here, but
+    // do not do so.
+    //
+    // The problem is that if we do so, C++ will think we have
+    // seralisers for *all* types. We don't, of course, but that won't
+    // cause an error until link time. That makes for long debug cycles
+    // when there is a missing serialiser.
+    //
+    // By not defining it globally, and instead letting individual
+    // serialisers specialise the type, we get back the compile-time
+    // errors we would like. When no serialiser exists, C++ sees an
+    // abstract "incomplete" type with no definition, and any attempt to
+    // use `to` or `from` static methods is a compile-time error because
+    // they don't exist on an incomplete type.
+    //
+    // This makes for a quicker debug cycle, as desired.
+#if 0
+    {
+        static T read(const Store & store, Source & from);
+        static void write(const Store & store, Sink & out, const T & t);
+    };
+#endif
+
+    /**
+     * Wrapper function around `WorkerProto::Serialise<T>::write` that allows us to
+     * infer the type instead of having to write it down explicitly.
+     */
+    template<typename T>
+    static void write(const Store & store, Sink & out, const T & t)
+    {
+        WorkerProto::Serialise<T>::write(store, out, t);
+    }
+};
+
+enum struct WorkerProto::Op : uint64_t
+{
+    IsValidPath = 1,
+    HasSubstitutes = 3,
+    QueryPathHash = 4, // obsolete
+    QueryReferences = 5, // obsolete
+    QueryReferrers = 6,
+    AddToStore = 7,
+    AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore
+    BuildPaths = 9,
+    EnsurePath = 10,
+    AddTempRoot = 11,
+    AddIndirectRoot = 12,
+    SyncWithGC = 13,
+    FindRoots = 14,
+    ExportPath = 16, // obsolete
+    QueryDeriver = 18, // obsolete
+    SetOptions = 19,
+    CollectGarbage = 20,
+    QuerySubstitutablePathInfo = 21,
+    QueryDerivationOutputs = 22, // obsolete
+    QueryAllValidPaths = 23,
+    QueryFailedPaths = 24,
+    ClearFailedPaths = 25,
+    QueryPathInfo = 26,
+    ImportPaths = 27, // obsolete
+    QueryDerivationOutputNames = 28, // obsolete
+    QueryPathFromHashPart = 29,
+    QuerySubstitutablePathInfos = 30,
+    QueryValidPaths = 31,
+    QuerySubstitutablePaths = 32,
+    QueryValidDerivers = 33,
+    OptimiseStore = 34,
+    VerifyStore = 35,
+    BuildDerivation = 36,
+    AddSignatures = 37,
+    NarFromPath = 38,
+    AddToStoreNar = 39,
+    QueryMissing = 40,
+    QueryDerivationOutputMap = 41,
+    RegisterDrvOutput = 42,
+    QueryRealisation = 43,
+    AddMultipleToStore = 44,
+    AddBuildLog = 45,
+    BuildPathsWithResults = 46,
 };
 
 /**
- * Wrapper function around `WorkerProto<T>::write` that allows us to
- * infer the type instead of having to write it down explicitly.
+ * Convenience for sending operation codes.
+ *
+ * @todo Switch to using `WorkerProto::Serialise` instead probably. But
+ * this was not done at this time so there would be less churn.
  */
-template<typename T>
-void workerProtoWrite(const Store & store, Sink & out, const T & t)
+inline Sink & operator << (Sink & sink, WorkerProto::Op op)
 {
-    WorkerProto<T>::write(store, out, t);
+    return sink << (uint64_t) op;
 }
 
 /**
- * Declare a canonical serializer pair for the worker protocol.
+ * Convenience for debugging.
  *
- * We specialize the struct merely to indicate that we are implementing
+ * @todo Perhaps render known opcodes more nicely.
+ */
+inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
+{
+    return s << (uint64_t) op;
+}
+
+/**
+ * Declare a canonical serialiser pair for the worker protocol.
+ *
+ * We specialise the struct merely to indicate that we are implementing
  * the function for the given type.
  *
  * Some sort of `template<...>` must be used with the caller for this to
@@ -120,7 +170,7 @@ void workerProtoWrite(const Store & store, Sink & out, const T & t)
  * practice.
  */
 #define MAKE_WORKER_PROTO(T) \
-    struct WorkerProto< T > { \
+    struct WorkerProto::Serialise< T > { \
         static T read(const Store & store, Source & from); \
         static void write(const Store & store, Sink & out, const T & t); \
     };
@@ -156,7 +206,7 @@ MAKE_WORKER_PROTO(X_);
 
 /**
  * These use the empty string for the null case, relying on the fact
- * that the underlying types never serialize to the empty string.
+ * that the underlying types never serialise to the empty string.
  *
  * We do this instead of a generic std::optional<T> instance because
  * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 94368e415..1558ac11e 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -850,7 +850,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = WorkerProto<StorePathSet>::read(*store, in);
+                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -859,19 +859,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                     store->substitutePaths(paths);
                 }
 
-                workerProtoWrite(*store, out, store->queryValidPaths(paths));
+                WorkerProto::write(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                auto paths = WorkerProto<StorePathSet>::read(*store, in);
+                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        workerProtoWrite(*store, out, info->references);
+                        WorkerProto::write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -899,7 +899,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(WorkerProto<StorePathSet>::read(*store, in), out);
+                store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, in), out);
                 break;
             }
 
@@ -945,7 +945,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                     DrvOutputs builtOutputs;
                     for (auto & [output, realisation] : status.builtOutputs)
                         builtOutputs.insert_or_assign(realisation.id, realisation);
-                    workerProtoWrite(*store, out, builtOutputs);
+                    WorkerProto::write(*store, out, builtOutputs);
                 }
 
                 break;
@@ -954,9 +954,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(WorkerProto<StorePathSet>::read(*store, in),
+                store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, in),
                     closure, false, includeOutputs);
-                workerProtoWrite(*store, out, closure);
+                WorkerProto::write(*store, out, closure);
                 break;
             }
 
@@ -971,7 +971,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = WorkerProto<StorePathSet>::read(*store, in);
+                info.references = WorkerProto::Serialise<StorePathSet>::read(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));

From 4e8b495ad7dddabc35bf9d6afe3573426ffed15d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 26 May 2023 11:22:24 -0400
Subject: [PATCH 120/183] Likewise namespace and `enum struct`-ify
 `ServeCommand`

The motivation is exactly the same as for the last commit. In addition,
this anticipates us formally defining separate serialisers for the serve
protocol.
---
 src/libstore/legacy-ssh-store.cc | 16 ++++-----
 src/libstore/serve-protocol.hh   | 58 ++++++++++++++++++++++++++------
 src/nix-store/nix-store.cc       | 22 ++++++------
 3 files changed, 66 insertions(+), 30 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 7d006f451..55ecab0ff 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -134,7 +134,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
 
             debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
 
-            conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)};
+            conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
             conn->to.flush();
 
             auto p = readString(conn->from);
@@ -177,7 +177,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
 
             conn->to
-                << cmdAddToStoreNar
+                << ServeProto::Command::AddToStoreNar
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
@@ -199,7 +199,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         } else {
 
             conn->to
-                << cmdImportPaths
+                << ServeProto::Command::ImportPaths
                 << 1;
             try {
                 copyNAR(source, conn->to);
@@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     {
         auto conn(connections->get());
 
-        conn->to << cmdDumpStorePath << printStorePath(path);
+        conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
         conn->to.flush();
         copyNAR(conn->from, sink);
     }
@@ -280,7 +280,7 @@ public:
         auto conn(connections->get());
 
         conn->to
-            << cmdBuildDerivation
+            << ServeProto::Command::BuildDerivation
             << printStorePath(drvPath);
         writeDerivation(conn->to, *this, drv);
 
@@ -311,7 +311,7 @@ public:
 
         auto conn(connections->get());
 
-        conn->to << cmdBuildPaths;
+        conn->to << ServeProto::Command::BuildPaths;
         Strings ss;
         for (auto & p : drvPaths) {
             auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
@@ -368,7 +368,7 @@ public:
         auto conn(connections->get());
 
         conn->to
-            << cmdQueryClosure
+            << ServeProto::Command::QueryClosure
             << includeOutputs;
         WorkerProto::write(*this, conn->to, paths);
         conn->to.flush();
@@ -383,7 +383,7 @@ public:
         auto conn(connections->get());
 
         conn->to
-            << cmdQueryValidPaths
+            << ServeProto::Command::QueryValidPaths
             << false // lock
             << maybeSubstitute;
         WorkerProto::write(*this, conn->to, paths);
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 553fd3a09..7e43b3969 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -10,16 +10,52 @@ namespace nix {
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
-typedef enum {
-    cmdQueryValidPaths = 1,
-    cmdQueryPathInfos = 2,
-    cmdDumpStorePath = 3,
-    cmdImportPaths = 4,
-    cmdExportPaths = 5,
-    cmdBuildPaths = 6,
-    cmdQueryClosure = 7,
-    cmdBuildDerivation = 8,
-    cmdAddToStoreNar = 9,
-} ServeCommand;
+/**
+ * The "serve protocol", used by ssh:// stores.
+ *
+ * This `struct` is basically just a `namespace`; We use a type rather
+ * than a namespace just so we can use it as a template argument.
+ */
+struct ServeProto
+{
+    /**
+     * Enumeration of all the request types for the protocol.
+     */
+    enum struct Command : uint64_t;
+};
+
+enum struct ServeProto::Command : uint64_t
+{
+    QueryValidPaths = 1,
+    QueryPathInfos = 2,
+    DumpStorePath = 3,
+    ImportPaths = 4,
+    ExportPaths = 5,
+    BuildPaths = 6,
+    QueryClosure = 7,
+    BuildDerivation = 8,
+    AddToStoreNar = 9,
+};
+
+/**
+ * Convenience for sending operation codes.
+ *
+ * @todo Switch to using `ServeProto::Serialize` instead probably. But
+ * this was not done at this time so there would be less churn.
+ */
+inline Sink & operator << (Sink & sink, ServeProto::Command op)
+{
+    return sink << (uint64_t) op;
+}
+
+/**
+ * Convenience for debugging.
+ *
+ * @todo Perhaps render known opcodes more nicely.
+ */
+inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
+{
+    return s << (uint64_t) op;
+}
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 1558ac11e..062c68ff8 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -838,16 +838,16 @@ static void opServe(Strings opFlags, Strings opArgs)
     };
 
     while (true) {
-        ServeCommand cmd;
+        ServeProto::Command cmd;
         try {
-            cmd = (ServeCommand) readInt(in);
+            cmd = (ServeProto::Command) readInt(in);
         } catch (EndOfFile & e) {
             break;
         }
 
         switch (cmd) {
 
-            case cmdQueryValidPaths: {
+            case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
                 auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
@@ -863,7 +863,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdQueryPathInfos: {
+            case ServeProto::Command::QueryPathInfos: {
                 auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
@@ -886,24 +886,24 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdDumpStorePath:
+            case ServeProto::Command::DumpStorePath:
                 store->narFromPath(store->parseStorePath(readString(in)), out);
                 break;
 
-            case cmdImportPaths: {
+            case ServeProto::Command::ImportPaths: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
                 store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
                 out << 1; // indicate success
                 break;
             }
 
-            case cmdExportPaths: {
+            case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
                 store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, in), out);
                 break;
             }
 
-            case cmdBuildPaths: {
+            case ServeProto::Command::BuildPaths: {
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
 
@@ -924,7 +924,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdBuildDerivation: { /* Used by hydra-queue-runner. */
+            case ServeProto::Command::BuildDerivation: { /* Used by hydra-queue-runner. */
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
 
@@ -951,7 +951,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdQueryClosure: {
+            case ServeProto::Command::QueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
                 store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, in),
@@ -960,7 +960,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdAddToStoreNar: {
+            case ServeProto::Command::AddToStoreNar: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
 
                 auto path = readString(in);

From 9f69b7dee9fc6035b8aa0cc718f5e74af460d9aa Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 17 Apr 2023 13:40:46 -0400
Subject: [PATCH 121/183] Create `worker_proto::{Read,Write}Conn`

Pass this around instead of `Source &` and `Sink &` directly. This will
give us something to put the protocol version on once the time comes.

To do this ergonomically, we need to expose `RemoteStore::Connection`,
so do that too. Give it some more API docs while we are at it.
---
 src/libstore/build/derivation-goal.cc   |   6 +-
 src/libstore/daemon.cc                  |  64 ++++++++-------
 src/libstore/derivations.cc             |   7 +-
 src/libstore/export-import.cc           |   7 +-
 src/libstore/legacy-ssh-store.cc        |  52 ++++++++++--
 src/libstore/path-info.cc               |   7 +-
 src/libstore/remote-store-connection.hh |  97 ++++++++++++++++++++++
 src/libstore/remote-store.cc            |  76 +++++++++---------
 src/libstore/remote-store.hh            |  16 +---
 src/libstore/ssh-store.cc               |   1 +
 src/libstore/uds-remote-store.hh        |   1 +
 src/libstore/worker-protocol-impl.hh    |  40 +++++-----
 src/libstore/worker-protocol.cc         | 102 ++++++++++++------------
 src/libstore/worker-protocol.hh         |  34 ++++++--
 src/nix-store/nix-store.cc              |  21 ++---
 src/nix/daemon.cc                       |   1 +
 16 files changed, 348 insertions(+), 184 deletions(-)
 create mode 100644 src/libstore/remote-store-connection.hh

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 1a946e3d2..5e37f7ecb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -1151,9 +1151,11 @@ HookReply DerivationGoal::tryBuildHook()
         throw;
     }
 
+    WorkerProto::WriteConn conn { hook->sink };
+
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    WorkerProto::write(worker.store, hook->sink, inputPaths);
+    WorkerProto::write(worker.store, conn, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1164,7 +1166,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        WorkerProto::write(worker.store, hook->sink, missingOutputs);
+        WorkerProto::write(worker.store, conn, missingOutputs);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 7eba1a79d..75c3d2aca 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -260,13 +260,13 @@ struct ClientSettings
     }
 };
 
-static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
+static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn)
 {
     std::vector<DerivedPath> reqs;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
-        reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, from);
+        reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, conn);
     } else {
-        for (auto & s : readStrings<Strings>(from))
+        for (auto & s : readStrings<Strings>(conn.from))
             reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
     }
     return reqs;
@@ -276,6 +276,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
     Source & from, BufferedSink & to, WorkerProto::Op op)
 {
+    WorkerProto::ReadConn rconn { .from = from };
+    WorkerProto::WriteConn wconn { .to = to };
+
     switch (op) {
 
     case WorkerProto::Op::IsValidPath: {
@@ -288,7 +291,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case WorkerProto::Op::QueryValidPaths: {
-        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
 
         SubstituteFlag substitute = NoSubstitute;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@@ -301,7 +304,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
-        WorkerProto::write(*store, to, res);
+        WorkerProto::write(*store, wconn, res);
         break;
     }
 
@@ -317,11 +320,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case WorkerProto::Op::QuerySubstitutablePaths: {
-        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+        auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        WorkerProto::write(*store, to, res);
+        WorkerProto::write(*store, wconn, res);
         break;
     }
 
@@ -350,7 +353,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        WorkerProto::write(*store, to, paths);
+        WorkerProto::write(*store, wconn, paths);
         break;
     }
 
@@ -368,7 +371,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        WorkerProto::write(*store, to, outputs);
+        WorkerProto::write(*store, wconn, outputs);
         break;
     }
 
@@ -394,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+            auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -496,7 +499,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::AddTextToStore: {
         std::string suffix = readString(from);
         std::string s = readString(from);
-        auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+        auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -528,7 +531,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case WorkerProto::Op::BuildPaths: {
-        auto drvs = readDerivedPaths(*store, clientVersion, from);
+        auto drvs = readDerivedPaths(*store, clientVersion, rconn);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -553,7 +556,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case WorkerProto::Op::BuildPathsWithResults: {
-        auto drvs = readDerivedPaths(*store, clientVersion, from);
+        auto drvs = readDerivedPaths(*store, clientVersion, rconn);
         BuildMode mode = bmNormal;
         mode = (BuildMode) readInt(from);
 
@@ -568,7 +571,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto results = store->buildPathsWithResults(drvs, mode);
         logger->stopWork();
 
-        WorkerProto::write(*store, to, results);
+        WorkerProto::write(*store, wconn, results);
 
         break;
     }
@@ -645,7 +648,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             DrvOutputs builtOutputs;
             for (auto & [output, realisation] : res.builtOutputs)
                 builtOutputs.insert_or_assign(realisation.id, realisation);
-            WorkerProto::write(*store, to, builtOutputs);
+            WorkerProto::write(*store, wconn, builtOutputs);
         }
         break;
     }
@@ -710,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::CollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+        options.pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -780,7 +783,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            WorkerProto::write(*store, to, i->second.references);
+            WorkerProto::write(*store, wconn, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -791,11 +794,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+            auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = WorkerProto::Serialise<StorePathCAMap>::read(*store, from);
+            pathsMap = WorkerProto::Serialise<StorePathCAMap>::read(*store, rconn);
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -803,7 +806,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            WorkerProto::write(*store, to, i.second.references);
+            WorkerProto::write(*store, wconn, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -813,7 +816,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        WorkerProto::write(*store, to, paths);
+        WorkerProto::write(*store, wconn, paths);
         break;
     }
 
@@ -885,7 +888,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = WorkerProto::Serialise<StorePathSet>::read(*store, from);
+        info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = ContentAddress::parseOpt(readString(from));
@@ -930,15 +933,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case WorkerProto::Op::QueryMissing: {
-        auto targets = readDerivedPaths(*store, clientVersion, from);
+        auto targets = readDerivedPaths(*store, clientVersion, rconn);
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        WorkerProto::write(*store, to, willBuild);
-        WorkerProto::write(*store, to, willSubstitute);
-        WorkerProto::write(*store, to, unknown);
+        WorkerProto::write(*store, wconn, willBuild);
+        WorkerProto::write(*store, wconn, willSubstitute);
+        WorkerProto::write(*store, wconn, unknown);
         to << downloadSize << narSize;
         break;
     }
@@ -951,7 +954,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             store->registerDrvOutput(Realisation{
                 .id = outputId, .outPath = outputPath});
         } else {
-            auto realisation = WorkerProto::Serialise<Realisation>::read(*store, from);
+            auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
             store->registerDrvOutput(realisation);
         }
         logger->stopWork();
@@ -966,11 +969,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             std::set<StorePath> outPaths;
             if (info) outPaths.insert(info->outPath);
-            WorkerProto::write(*store, to, outPaths);
+            WorkerProto::write(*store, wconn, outPaths);
         } else {
             std::set<Realisation> realisations;
             if (info) realisations.insert(*info);
-            WorkerProto::write(*store, to, realisations);
+            WorkerProto::write(*store, wconn, realisations);
         }
         break;
     }
@@ -1050,7 +1053,8 @@ void processConnection(
         auto temp = trusted
             ? store->isTrustedClient()
             : std::optional { NotTrusted };
-        WorkerProto::write(*store, to, temp);
+        WorkerProto::WriteConn wconn { .to = to };
+        WorkerProto::write(*store, wconn, temp);
     }
 
     /* Send startup error messages to the client. */
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 2295ff9e8..6f63685d4 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -750,7 +750,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store, in);
+    drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store,
+        WorkerProto::ReadConn { .from = in });
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -798,7 +799,9 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.raw());
     }
-    WorkerProto::write(store, out, drv.inputSrcs);
+    WorkerProto::write(store,
+        WorkerProto::WriteConn { .to = out },
+        drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index b4100bace..e866aeb42 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -46,7 +46,9 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    WorkerProto::write(*this, teeSink, info->references);
+    WorkerProto::write(*this,
+        WorkerProto::WriteConn { .to = teeSink },
+        info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -74,7 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
 
-        auto references = WorkerProto::Serialise<StorePathSet>::read(*this, source);
+        auto references = WorkerProto::Serialise<StorePathSet>::read(*this,
+            WorkerProto::ReadConn { .from = source });
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 55ecab0ff..fa17d606d 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -48,6 +48,42 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         FdSource from;
         int remoteVersion;
         bool good = true;
+
+        /**
+         * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
+         * factored out worker protocol searlizers with a
+         * `LegacySSHStore::Connection`.
+         *
+         * The worker protocol connection types are unidirectional, unlike
+         * this type.
+         *
+         * @todo Use server protocol serializers, not worker protocol
+         * serializers, once we have made that distiction.
+         */
+        operator WorkerProto::ReadConn ()
+        {
+            return WorkerProto::ReadConn {
+                .from = from,
+            };
+        }
+
+        /*
+         * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
+         * factored out worker protocol searlizers with a
+         * `LegacySSHStore::Connection`.
+         *
+         * The worker protocol connection types are unidirectional, unlike
+         * this type.
+         *
+         * @todo Use server protocol serializers, not worker protocol
+         * serializers, once we have made that distiction.
+         */
+        operator WorkerProto::WriteConn ()
+        {
+            return WorkerProto::WriteConn {
+                .to = to,
+            };
+        }
     };
 
     std::string host;
@@ -147,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+            info->references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -181,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            WorkerProto::write(*this, conn->to, info.references);
+            WorkerProto::write(*this, *conn, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -210,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            WorkerProto::write(*this, conn->to, info.references);
+            WorkerProto::write(*this, *conn, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -295,7 +331,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, conn->from);
+            auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -370,10 +406,10 @@ public:
         conn->to
             << ServeProto::Command::QueryClosure
             << includeOutputs;
-        WorkerProto::write(*this, conn->to, paths);
+        WorkerProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, conn->from))
+        for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn))
             out.insert(i);
     }
 
@@ -386,10 +422,10 @@ public:
             << ServeProto::Command::QueryValidPaths
             << false // lock
             << maybeSubstitute;
-        WorkerProto::write(*this, conn->to, paths);
+        WorkerProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
     }
 
     void connect() override
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 9971de1dd..981bbfb14 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -133,7 +133,8 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
     auto narHash = Hash::parseAny(readString(source), htSHA256);
     ValidPathInfo info(path, narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    info.references = WorkerProto::Serialise<StorePathSet>::read(store, source);
+    info.references = WorkerProto::Serialise<StorePathSet>::read(store,
+        WorkerProto::ReadConn { .from = source });
     source >> info.registrationTime >> info.narSize;
     if (format >= 16) {
         source >> info.ultimate;
@@ -154,7 +155,9 @@ void ValidPathInfo::write(
         sink << store.printStorePath(path);
     sink << (deriver ? store.printStorePath(*deriver) : "")
          << narHash.to_string(Base16, false);
-    WorkerProto::write(store, sink, references);
+    WorkerProto::write(store,
+        WorkerProto::WriteConn { .to = sink },
+        references);
     sink << registrationTime << narSize;
     if (format >= 16) {
         sink << ultimate
diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh
new file mode 100644
index 000000000..d32d91a60
--- /dev/null
+++ b/src/libstore/remote-store-connection.hh
@@ -0,0 +1,97 @@
+#include "remote-store.hh"
+#include "worker-protocol.hh"
+
+namespace nix {
+
+/**
+ * Bidirectional connection (send and receive) used by the Remote Store
+ * implementation.
+ *
+ * Contains `Source` and `Sink` for actual communication, along with
+ * other information learned when negotiating the connection.
+ */
+struct RemoteStore::Connection
+{
+    /**
+     * Send with this.
+     */
+    FdSink to;
+
+    /**
+     * Receive with this.
+     */
+    FdSource from;
+
+    /**
+     * Worker protocol version used for the connection.
+     *
+     * Despite its name, I think it is actually the maximum version both
+     * sides support. (If the maximum doesn't exist, we would fail to
+     * establish a connection and produce a value of this type.)
+     */
+    unsigned int daemonVersion;
+
+    /**
+     * Whether the remote side trusts us or not.
+     *
+     * 3 values: "yes", "no", or `std::nullopt` for "unknown".
+     *
+     * Note that the "remote side" might not be just the end daemon, but
+     * also an intermediary forwarder that can make its own trusting
+     * decisions. This would be the intersection of all their trust
+     * decisions, since it takes only one link in the chain to start
+     * denying operations.
+     */
+    std::optional<TrustedFlag> remoteTrustsUs;
+
+    /**
+     * The version of the Nix daemon that is processing our requests.
+     *
+     * Do note, it may or may not communicating with another daemon,
+     * rather than being an "end" `LocalStore` or similar.
+     */
+    std::optional<std::string> daemonNixVersion;
+
+    /**
+     * Time this connection was established.
+     */
+    std::chrono::time_point<std::chrono::steady_clock> startTime;
+
+    /**
+     * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
+     * factored out worker protocol searlizers with a
+     * `RemoteStore::Connection`.
+     *
+     * The worker protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator WorkerProto::ReadConn ()
+    {
+        return WorkerProto::ReadConn {
+            .from = from,
+        };
+    }
+
+    /**
+     * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
+     * factored out worker protocol searlizers with a
+     * `RemoteStore::Connection`.
+     *
+     * The worker protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator WorkerProto::WriteConn ()
+    {
+        return WorkerProto::WriteConn {
+            .to = to,
+        };
+    }
+
+    virtual ~Connection();
+
+    virtual void closeWrite() = 0;
+
+    std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
+};
+
+}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 9253ce09e..1e2104e1f 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -5,6 +5,7 @@
 #include "remote-fs-accessor.hh"
 #include "build-result.hh"
 #include "remote-store.hh"
+#include "remote-store-connection.hh"
 #include "worker-protocol.hh"
 #include "worker-protocol-impl.hh"
 #include "archive.hh"
@@ -101,7 +102,7 @@ void RemoteStore::initConnection(Connection & conn)
         }
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
-            conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn.from);
+            conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn);
         } else {
             // We don't know the answer; protocol to old.
             conn.remoteTrustsUs = std::nullopt;
@@ -185,6 +186,7 @@ struct ConnectionHandle
     }
 
     RemoteStore::Connection * operator -> () { return &*handle; }
+    RemoteStore::Connection & operator * () { return *handle; }
 
     void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
     {
@@ -228,12 +230,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << WorkerProto::Op::QueryValidPaths;
-        WorkerProto::write(*this, conn->to, paths);
+        WorkerProto::write(*this, *conn, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
             conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
         }
         conn.processStderr();
-        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
     }
 }
 
@@ -243,7 +245,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryAllValidPaths;
     conn.processStderr();
-    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
 }
 
 
@@ -260,9 +262,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << WorkerProto::Op::QuerySubstitutablePaths;
-        WorkerProto::write(*this, conn->to, paths);
+        WorkerProto::write(*this, *conn, paths);
         conn.processStderr();
-        return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
     }
 }
 
@@ -284,7 +286,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -297,9 +299,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            WorkerProto::write(*this, conn->to, paths);
+            WorkerProto::write(*this, *conn, paths);
         } else
-            WorkerProto::write(*this, conn->to, pathsMap);
+            WorkerProto::write(*this, *conn, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -307,7 +309,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -350,7 +352,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, conn->from))
+    for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn))
         referrers.insert(i);
 }
 
@@ -360,7 +362,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
 }
 
 
@@ -372,7 +374,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     auto conn(getConnection());
     conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+    return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
 }
 
 
@@ -382,7 +384,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
         auto conn(getConnection());
         conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
         conn.processStderr();
-        return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
+        return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn);
     } else {
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -428,7 +430,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
             << WorkerProto::Op::AddToStore
             << name
             << caMethod.render(hashType);
-        WorkerProto::write(*this, conn->to, references);
+        WorkerProto::write(*this, *conn, references);
         conn->to << repair;
 
         // The dump source may invoke the store, so we need to make some room.
@@ -453,7 +455,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                         name, printHashType(hashType));
                 std::string s = dump.drain();
                 conn->to << WorkerProto::Op::AddTextToStore << name << s;
-                WorkerProto::write(*this, conn->to, references);
+                WorkerProto::write(*this, *conn, references);
                 conn.processStderr();
             },
             [&](const FileIngestionMethod & fim) -> void {
@@ -519,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            WorkerProto::write(*this, sink, info.references);
+            WorkerProto::write(*this, *conn, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -529,7 +531,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
         assert(importedPaths.size() <= 1);
     }
 
@@ -538,7 +540,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        WorkerProto::write(*this, conn->to, info.references);
+        WorkerProto::write(*this, *conn, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -611,7 +613,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
         conn->to << info.id.to_string();
         conn->to << std::string(info.outPath.to_string());
     } else {
-        WorkerProto::write(*this, conn->to, info);
+        WorkerProto::write(*this, *conn, info);
     }
     conn.processStderr();
 }
@@ -634,13 +636,13 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
         auto real = [&]() -> std::shared_ptr<const Realisation> {
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
                 auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read(
-                    *this, conn->from);
+                    *this, *conn);
                 if (outPaths.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
             } else {
                 auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(
-                    *this, conn->from);
+                    *this, *conn);
                 if (realisations.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(*realisations.begin());
@@ -651,10 +653,10 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
     } catch (...) { return callback.rethrow(); }
 }
 
-static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
+static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector<DerivedPath> & reqs)
 {
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
-        WorkerProto::write(store, conn->to, reqs);
+    if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) {
+        WorkerProto::write(store, conn, reqs);
     } else {
         Strings ss;
         for (auto & p : reqs) {
@@ -666,12 +668,12 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons
                 [&](const StorePath & drvPath) {
                     throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
                         store.printStorePath(drvPath),
-                        GET_PROTOCOL_MAJOR(conn->daemonVersion),
-                        GET_PROTOCOL_MINOR(conn->daemonVersion));
+                        GET_PROTOCOL_MAJOR(conn.daemonVersion),
+                        GET_PROTOCOL_MINOR(conn.daemonVersion));
                 },
             }, sOrDrvPath);
         }
-        conn->to << ss;
+        conn.to << ss;
     }
 }
 
@@ -697,7 +699,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
     auto conn(getConnection());
     conn->to << WorkerProto::Op::BuildPaths;
     assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
-    writeDerivedPaths(*this, conn, drvPaths);
+    writeDerivedPaths(*this, *conn, drvPaths);
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
         conn->to << buildMode;
     else
@@ -721,10 +723,10 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
         conn->to << WorkerProto::Op::BuildPathsWithResults;
-        writeDerivedPaths(*this, conn, paths);
+        writeDerivedPaths(*this, *conn, paths);
         conn->to << buildMode;
         conn.processStderr();
-        return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, conn->from);
+        return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
     } else {
         // Avoid deadlock.
         conn_.reset();
@@ -807,7 +809,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
         conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
-        auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, conn->from);
+        auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
         for (auto && [output, realisation] : builtOutputs)
             res.builtOutputs.insert_or_assign(
                 std::move(output.outputName),
@@ -866,7 +868,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << WorkerProto::Op::CollectGarbage << options.action;
-    WorkerProto::write(*this, conn->to, options.pathsToDelete);
+    WorkerProto::write(*this, *conn, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -923,11 +925,11 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
             // to prevent a deadlock.
             goto fallback;
         conn->to << WorkerProto::Op::QueryMissing;
-        writeDerivedPaths(*this, conn, targets);
+        writeDerivedPaths(*this, *conn, targets);
         conn.processStderr();
-        willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
-        willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
-        unknown = WorkerProto::Serialise<StorePathSet>::read(*this, conn->from);
+        willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+        willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+        unknown = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4f3971bfd..cb7a71acf 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -166,21 +166,7 @@ public:
 
     void flushBadConnections();
 
-    struct Connection
-    {
-        FdSink to;
-        FdSource from;
-        unsigned int daemonVersion;
-        std::optional<TrustedFlag> remoteTrustsUs;
-        std::optional<std::string> daemonNixVersion;
-        std::chrono::time_point<std::chrono::steady_clock> startTime;
-
-        virtual ~Connection();
-
-        virtual void closeWrite() = 0;
-
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
-    };
+    struct Connection;
 
     ref<Connection> openConnectionWrapper();
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 962221ad2..0200076c0 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -1,6 +1,7 @@
 #include "ssh-store-config.hh"
 #include "store-api.hh"
 #include "remote-store.hh"
+#include "remote-store-connection.hh"
 #include "remote-fs-accessor.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index 3bbab371c..2bd6517fa 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "remote-store.hh"
+#include "remote-store-connection.hh"
 #include "local-fs-store.hh"
 
 namespace nix {
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
index f34e55c54..d3d2792ff 100644
--- a/src/libstore/worker-protocol-impl.hh
+++ b/src/libstore/worker-protocol-impl.hh
@@ -13,65 +13,65 @@
 namespace nix {
 
 template<typename T>
-std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, Source & from)
+std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     std::vector<T> resSet;
-    auto size = readNum<size_t>(from);
+    auto size = readNum<size_t>(conn.from);
     while (size--) {
-        resSet.push_back(WorkerProto::Serialise<T>::read(store, from));
+        resSet.push_back(WorkerProto::Serialise<T>::read(store, conn));
     }
     return resSet;
 }
 
 template<typename T>
-void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
+void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector<T> & resSet)
 {
-    out << resSet.size();
+    conn.to << resSet.size();
     for (auto & key : resSet) {
-        WorkerProto::Serialise<T>::write(store, out, key);
+        WorkerProto::Serialise<T>::write(store, conn, key);
     }
 }
 
 template<typename T>
-std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, Source & from)
+std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     std::set<T> resSet;
-    auto size = readNum<size_t>(from);
+    auto size = readNum<size_t>(conn.from);
     while (size--) {
-        resSet.insert(WorkerProto::Serialise<T>::read(store, from));
+        resSet.insert(WorkerProto::Serialise<T>::read(store, conn));
     }
     return resSet;
 }
 
 template<typename T>
-void WorkerProto::Serialise<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
+void WorkerProto::Serialise<std::set<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::set<T> & resSet)
 {
-    out << resSet.size();
+    conn.to << resSet.size();
     for (auto & key : resSet) {
-        WorkerProto::Serialise<T>::write(store, out, key);
+        WorkerProto::Serialise<T>::write(store, conn, key);
     }
 }
 
 template<typename K, typename V>
-std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, Source & from)
+std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     std::map<K, V> resMap;
-    auto size = readNum<size_t>(from);
+    auto size = readNum<size_t>(conn.from);
     while (size--) {
-        auto k = WorkerProto::Serialise<K>::read(store, from);
-        auto v = WorkerProto::Serialise<V>::read(store, from);
+        auto k = WorkerProto::Serialise<K>::read(store, conn);
+        auto v = WorkerProto::Serialise<V>::read(store, conn);
         resMap.insert_or_assign(std::move(k), std::move(v));
     }
     return resMap;
 }
 
 template<typename K, typename V>
-void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, WorkerProto::WriteConn conn, const std::map<K, V> & resMap)
 {
-    out << resMap.size();
+    conn.to << resMap.size();
     for (auto & i : resMap) {
-        WorkerProto::Serialise<K>::write(store, out, i.first);
-        WorkerProto::Serialise<V>::write(store, out, i.second);
+        WorkerProto::Serialise<K>::write(store, conn, i.first);
+        WorkerProto::Serialise<V>::write(store, conn, i.second);
     }
 }
 
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 0bf9e4d68..a23130743 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -12,31 +12,31 @@
 
 namespace nix {
 
-std::string WorkerProto::Serialise<std::string>::read(const Store & store, Source & from)
+std::string WorkerProto::Serialise<std::string>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    return readString(from);
+    return readString(conn.from);
 }
 
-void WorkerProto::Serialise<std::string>::write(const Store & store, Sink & out, const std::string & str)
+void WorkerProto::Serialise<std::string>::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str)
 {
-    out << str;
+    conn.to << str;
 }
 
 
-StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, Source & from)
+StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    return store.parseStorePath(readString(from));
+    return store.parseStorePath(readString(conn.from));
 }
 
-void WorkerProto::Serialise<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
+void WorkerProto::Serialise<StorePath>::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath)
 {
-    out << store.printStorePath(storePath);
+    conn.to << store.printStorePath(storePath);
 }
 
 
-std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
+std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    auto temp = readNum<uint8_t>(from);
+    auto temp = readNum<uint8_t>(conn.from);
     switch (temp) {
         case 0:
             return std::nullopt;
@@ -49,17 +49,17 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
     }
 }
 
-void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
-        out << (uint8_t)0;
+        conn.to << (uint8_t)0;
     else {
         switch (*optTrusted) {
         case Trusted:
-            out << (uint8_t)1;
+            conn.to << (uint8_t)1;
             break;
         case NotTrusted:
-            out << (uint8_t)2;
+            conn.to << (uint8_t)2;
             break;
         default:
             assert(false);
@@ -68,83 +68,83 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
 }
 
 
-ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, Source & from)
+ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    return ContentAddress::parse(readString(from));
+    return ContentAddress::parse(readString(conn.from));
 }
 
-void WorkerProto::Serialise<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
+void WorkerProto::Serialise<ContentAddress>::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca)
 {
-    out << renderContentAddress(ca);
+    conn.to << renderContentAddress(ca);
 }
 
 
-DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Source & from)
+DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    auto s = readString(from);
+    auto s = readString(conn.from);
     return DerivedPath::parseLegacy(store, s);
 }
 
-void WorkerProto::Serialise<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
+void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
 {
-    out << req.to_string_legacy(store);
+    conn.to << req.to_string_legacy(store);
 }
 
 
-Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, Source & from)
+Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    std::string rawInput = readString(from);
+    std::string rawInput = readString(conn.from);
     return Realisation::fromJSON(
         nlohmann::json::parse(rawInput),
         "remote-protocol"
     );
 }
 
-void WorkerProto::Serialise<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
+void WorkerProto::Serialise<Realisation>::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation)
 {
-    out << realisation.toJSON().dump();
+    conn.to << realisation.toJSON().dump();
 }
 
 
-DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, Source & from)
+DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    return DrvOutput::parse(readString(from));
+    return DrvOutput::parse(readString(conn.from));
 }
 
-void WorkerProto::Serialise<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+void WorkerProto::Serialise<DrvOutput>::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput)
 {
-    out << drvOutput.to_string();
+    conn.to << drvOutput.to_string();
 }
 
 
-KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, Source & from)
+KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    auto path = WorkerProto::Serialise<DerivedPath>::read(store, from);
-    auto br = WorkerProto::Serialise<BuildResult>::read(store, from);
+    auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
+    auto br = WorkerProto::Serialise<BuildResult>::read(store, conn);
     return KeyedBuildResult {
         std::move(br),
         /* .path = */ std::move(path),
     };
 }
 
-void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
+void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
 {
-    WorkerProto::write(store, to, res.path);
-    WorkerProto::write(store, to, static_cast<const BuildResult &>(res));
+    WorkerProto::write(store, conn, res.path);
+    WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Source & from)
+BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     BuildResult res;
-    res.status = (BuildResult::Status) readInt(from);
-    from
+    res.status = (BuildResult::Status) readInt(conn.from);
+    conn.from
         >> res.errorMsg
         >> res.timesBuilt
         >> res.isNonDeterministic
         >> res.startTime
         >> res.stopTime;
-    auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, from);
+    auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn);
     for (auto && [output, realisation] : builtOutputs)
         res.builtOutputs.insert_or_assign(
             std::move(output.outputName),
@@ -152,9 +152,9 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Sourc
     return res;
 }
 
-void WorkerProto::Serialise<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
+void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
 {
-    to
+    conn.to
         << res.status
         << res.errorMsg
         << res.timesBuilt
@@ -164,30 +164,30 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, Sink & to,
     DrvOutputs builtOutputs;
     for (auto & [output, realisation] : res.builtOutputs)
         builtOutputs.insert_or_assign(realisation.id, realisation);
-    WorkerProto::write(store, to, builtOutputs);
+    WorkerProto::write(store, conn, builtOutputs);
 }
 
 
-std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, Source & from)
+std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    auto s = readString(from);
+    auto s = readString(conn.from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
 {
-    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+    conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, Source & from)
+std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
-    return ContentAddress::parseOpt(readString(from));
+    return ContentAddress::parseOpt(readString(conn.from));
 }
 
-void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
 {
-    out << (caOpt ? renderContentAddress(*caOpt) : "");
+    conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
 }
 
 }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index cd6801290..ff762c924 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -49,6 +49,28 @@ struct WorkerProto
      */
     enum struct Op : uint64_t;
 
+    /**
+     * A unidirectional read connection, to be used by the read half of the
+     * canonical serializers below.
+     *
+     * This currently is just a `Source &`, but more fields will be added
+     * later.
+     */
+    struct ReadConn {
+        Source & from;
+    };
+
+    /**
+     * A unidirectional write connection, to be used by the write half of the
+     * canonical serializers below.
+     *
+     * This currently is just a `Sink &`, but more fields will be added
+     * later.
+     */
+    struct WriteConn {
+        Sink & to;
+    };
+
     /**
      * Data type for canonical pairs of serialisers for the worker protocol.
      *
@@ -75,8 +97,8 @@ struct WorkerProto
     // This makes for a quicker debug cycle, as desired.
 #if 0
     {
-        static T read(const Store & store, Source & from);
-        static void write(const Store & store, Sink & out, const T & t);
+        static T read(const Store & store, ReadConn conn);
+        static void write(const Store & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -85,9 +107,9 @@ struct WorkerProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, Sink & out, const T & t)
+    static void write(const Store & store, WriteConn conn, const T & t)
     {
-        WorkerProto::Serialise<T>::write(store, out, t);
+        WorkerProto::Serialise<T>::write(store, conn, t);
     }
 };
 
@@ -171,8 +193,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
  */
 #define MAKE_WORKER_PROTO(T) \
     struct WorkerProto::Serialise< T > { \
-        static T read(const Store & store, Source & from); \
-        static void write(const Store & store, Sink & out, const T & t); \
+        static T read(const Store & store, WorkerProto::ReadConn conn); \
+        static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 062c68ff8..caa0248f1 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -807,6 +807,9 @@ static void opServe(Strings opFlags, Strings opArgs)
     out.flush();
     unsigned int clientVersion = readInt(in);
 
+    WorkerProto::ReadConn rconn { .from = in };
+    WorkerProto::WriteConn wconn { .to = out };
+
     auto getBuildSettings = [&]() {
         // FIXME: changing options here doesn't work if we're
         // building through the daemon.
@@ -850,7 +853,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
+                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -859,19 +862,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                     store->substitutePaths(paths);
                 }
 
-                WorkerProto::write(*store, out, store->queryValidPaths(paths));
+                WorkerProto::write(*store, wconn, store->queryValidPaths(paths));
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, in);
+                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        WorkerProto::write(*store, out, info->references);
+                        WorkerProto::write(*store, wconn, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -899,7 +902,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, in), out);
+                store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, rconn), out);
                 break;
             }
 
@@ -945,7 +948,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                     DrvOutputs builtOutputs;
                     for (auto & [output, realisation] : status.builtOutputs)
                         builtOutputs.insert_or_assign(realisation.id, realisation);
-                    WorkerProto::write(*store, out, builtOutputs);
+                    WorkerProto::write(*store, wconn, builtOutputs);
                 }
 
                 break;
@@ -954,9 +957,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, in),
+                store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, rconn),
                     closure, false, includeOutputs);
-                WorkerProto::write(*store, out, closure);
+                WorkerProto::write(*store, wconn, closure);
                 break;
             }
 
@@ -971,7 +974,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = WorkerProto::Serialise<StorePathSet>::read(*store, in);
+                info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 9fe9b3b1e..8e2bcf7e1 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -4,6 +4,7 @@
 #include "shared.hh"
 #include "local-store.hh"
 #include "remote-store.hh"
+#include "remote-store-connection.hh"
 #include "util.hh"
 #include "serialise.hh"
 #include "archive.hh"

From 3859cf6b21dddd7e9de729dc42b2f52d8523c239 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 19 Jun 2023 12:18:04 -0400
Subject: [PATCH 122/183] Remove unused `#include` from
 `local-derivation-goal.cc`

These were never needed for this file, and date back to before this was
split from `derivation-goal.cc`.
---
 src/libstore/build/local-derivation-goal.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 6e06f6061..09c1fcf08 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -10,8 +10,6 @@
 #include "archive.hh"
 #include "compression.hh"
 #include "daemon.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "json-utils.hh"

From 3910430b9d034c277650f4ff05a27008ede73c18 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Mon, 19 Jun 2023 21:00:49 +0200
Subject: [PATCH 123/183] Add more links in nix-build documentation (#8545)

* Add more links in nix-build documentation


Co-authored-by: John Ericson <git@JohnEricson.me>
---
 doc/manual/src/command-ref/nix-build.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index f70bbd7f2..7369ea2eb 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -51,8 +51,9 @@ derivation).
 
 # Options
 
-All options not listed here are passed to `nix-store --realise`,
-except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`.
+All options not listed here are passed to
+[`nix-store --realise`](nix-store/realise.md),
+except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
 
   - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
 

From 33d388983156e3cdd360a9bb87662da1b8a35f7a Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Mon, 19 Jun 2023 15:17:50 -0500
Subject: [PATCH 124/183] redirect old platform uninstall instruction links

Uninstall instructions were moved to their own page in #8267. The
overall section link was redirected in #8286, but platform-specific
links (which I give out frequently when I triage installer trouble)
weren't included.
---
 doc/manual/redirects.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index 5cd6fdea2..bddd56eb8 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -340,6 +340,8 @@ const redirects = {
     "attribute-sets": "#attribute-set"
   },
   "installation/installing-binary.html": {
+    "linux": "uninstall.html#linux",
+    "macos": "uninstall.html#macos",
     "uninstalling": "uninstall.html"
   }
 };

From a6a75ecad8c1a8bf264eb003b8c07c0fd66f80fb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Jun 2023 11:19:14 +0200
Subject: [PATCH 125/183] GC server: Clear O_NONBLOCK on the right file
 descriptor

The bug fix in 6d30f9e6fea7d451033653f8f167aef58f7f5347 erroneously
cleared O_NONBLOCK on the server rather than client FD (leaving both
in an incorrect state).

Fixes #8551.
---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0038ec802..d023bec71 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -563,7 +563,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                     /* On macOS, accepted sockets inherit the
                        non-blocking flag from the server socket, so
                        explicitly make it blocking. */
-                    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1)
+                    if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1)
                         abort();
 
                     while (true) {

From 6ae35534b7b6e10a26a0f2b2a0e37d7f7cfe47dd Mon Sep 17 00:00:00 2001
From: Ben Radford <104896700+benradf@users.noreply.github.com>
Date: Tue, 20 Jun 2023 10:34:09 +0100
Subject: [PATCH 126/183] Support opening local store with database on
 read-only filesystem (#8356)

Previously it was not possible to open a local store when its database is on a read-only filesystem. Obviously a store on a read-only filesystem cannot be modified, but it would still be useful to be able to query it.

This change adds a new read-only setting to LocalStore. When set to true, Nix will skip operations that fail when the database is on a read-only filesystem (acquiring big-lock, schema migration, etc), and the store database will be opened in immutable mode.

Co-authored-by: Ben Radford <benradf@users.noreply.github.com>
Co-authored-by: cidkidnix <cidkidnix@protonmail.com>
Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com>
Co-authored-by: John Ericson <git@JohnEricson.me>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libstore/gc.cc                   |  5 ++++
 src/libstore/local-store.cc          | 45 ++++++++++++++++++++++------
 src/libstore/local-store.hh          | 21 +++++++++++++
 src/libstore/sqlite.cc               | 11 ++++---
 src/libstore/sqlite.hh               | 23 +++++++++++++-
 src/libutil/experimental-features.cc |  9 +++++-
 src/libutil/experimental-features.hh |  1 +
 tests/local.mk                       |  3 +-
 tests/read-only-store.sh             | 42 ++++++++++++++++++++++++++
 9 files changed, 144 insertions(+), 16 deletions(-)
 create mode 100644 tests/read-only-store.sh

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0038ec802..3c9544017 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile()
 
 void LocalStore::addTempRoot(const StorePath & path)
 {
+    if (readOnly) {
+      debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
+      return;
+    }
+
     createTempRootsFile();
 
     /* Open/create the global GC lock file. */
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7fb312c37..e69460e6c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params)
 
     /* Create missing state directories if they don't already exist. */
     createDirs(realStoreDir);
-    makeStoreWritable();
+    if (readOnly) {
+        experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
+    } else {
+        makeStoreWritable();
+    }
     createDirs(linksDir);
     Path profilesDir = stateDir + "/profiles";
     createDirs(profilesDir);
@@ -204,8 +208,10 @@ LocalStore::LocalStore(const Params & params)
 
     for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
         createDirs(perUserDir);
-        if (chmod(perUserDir.c_str(), 0755) == -1)
-            throw SysError("could not set permissions on '%s' to 755", perUserDir);
+        if (!readOnly) {
+            if (chmod(perUserDir.c_str(), 0755) == -1)
+                throw SysError("could not set permissions on '%s' to 755", perUserDir);
+        }
     }
 
     /* Optionally, create directories and set permissions for a
@@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params)
 
     /* Acquire the big fat lock in shared mode to make sure that no
        schema upgrade is in progress. */
-    Path globalLockPath = dbDir + "/big-lock";
-    globalLock = openLockFile(globalLockPath.c_str(), true);
+    if (!readOnly) {
+        Path globalLockPath = dbDir + "/big-lock";
+        globalLock = openLockFile(globalLockPath.c_str(), true);
+    }
 
-    if (!lockFile(globalLock.get(), ltRead, false)) {
+    if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) {
         printInfo("waiting for the big Nix store lock...");
         lockFile(globalLock.get(), ltRead, true);
     }
@@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params)
     /* Check the current database schema and if necessary do an
        upgrade.  */
     int curSchema = getSchema();
+    if (readOnly && curSchema < nixSchemaVersion) {
+        debug("current schema version: %d", curSchema);
+        debug("supported schema version: %d", nixSchemaVersion);
+        throw Error(curSchema == 0 ?
+            "database does not exist, and cannot be created in read-only mode" :
+            "database schema needs migrating, but this cannot be done in read-only mode");
+    }
+
     if (curSchema > nixSchemaVersion)
         throw Error("current Nix store schema is version %1%, but I only support %2%",
              curSchema, nixSchemaVersion);
@@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params)
     else openDB(*state, false);
 
     if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-        migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
+        if (!readOnly) {
+            migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
+        } else {
+            throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode");
+        }
     }
 
     /* Prepare SQL statements. */
@@ -475,13 +495,20 @@ int LocalStore::getSchema()
 
 void LocalStore::openDB(State & state, bool create)
 {
-    if (access(dbDir.c_str(), R_OK | W_OK))
+    if (create && readOnly) {
+        throw Error("cannot create database while in read-only mode");
+    }
+
+    if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK)))
         throw SysError("Nix database directory '%1%' is not writable", dbDir);
 
     /* Open the Nix database. */
     std::string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
-    state.db = SQLite(dbPath, create);
+    auto openMode = readOnly ? SQLiteOpenMode::Immutable
+                  : create ? SQLiteOpenMode::Normal
+                  : SQLiteOpenMode::NoCreate;
+    state.db = SQLite(dbPath, openMode);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 70debad38..8a3b0b43f 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -46,6 +46,23 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
         "require-sigs",
         "Whether store paths copied into this store should have a trusted signature."};
 
+    Setting<bool> readOnly{(StoreConfig*) this,
+        false,
+        "read-only",
+        R"(
+          Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
+
+          Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
+
+          Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
+
+          > **Warning**
+          > Do not use this unless the filesystem is read-only.
+          >
+          > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
+          > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
+        )"};
+
     const std::string name() override { return "Local Store"; }
 
     std::string doc() override;
@@ -269,6 +286,10 @@ public:
 
 private:
 
+    /**
+     * Retrieve the current version of the database schema.
+     * If the database does not exist yet, the version returned will be 0.
+     */
     int getSchema();
 
     void openDB(State & state, bool create);
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index df334c23c..7c8decb74 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -1,6 +1,7 @@
 #include "sqlite.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "url.hh"
 
 #include <sqlite3.h>
 
@@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql)
     notice("SQL<[%1%]>", sql);
 };
 
-SQLite::SQLite(const Path & path, bool create)
+SQLite::SQLite(const Path & path, SQLiteOpenMode mode)
 {
     // useSQLiteWAL also indicates what virtual file system we need.  Using
     // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
     // for Linux (WSL) where useSQLiteWAL should be false by default.
     const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
-    int flags = SQLITE_OPEN_READWRITE;
-    if (create) flags |= SQLITE_OPEN_CREATE;
-    int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs);
+    bool immutable = mode == SQLiteOpenMode::Immutable;
+    int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE;
+    if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE;
+    auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0");
+    int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs);
     if (ret != SQLITE_OK) {
         const char * err = sqlite3_errstr(ret);
         throw Error("cannot open SQLite database '%s': %s", path, err);
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 6e14852cb..0c08267f7 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -11,6 +11,27 @@ struct sqlite3_stmt;
 
 namespace nix {
 
+enum class SQLiteOpenMode {
+    /**
+     * Open the database in read-write mode.
+     * If the database does not exist, it will be created.
+     */
+    Normal,
+    /**
+     * Open the database in read-write mode.
+     * Fails with an error if the database does not exist.
+     */
+    NoCreate,
+    /**
+     * Open the database in immutable mode.
+     * In addition to the database being read-only,
+     * no wal or journal files will be created by sqlite.
+     * Use this mode if the database is on a read-only filesystem.
+     * Fails with an error if the database does not exist.
+     */
+    Immutable,
+};
+
 /**
  * RAII wrapper to close a SQLite database automatically.
  */
@@ -18,7 +39,7 @@ struct SQLite
 {
     sqlite3 * db = 0;
     SQLite() { }
-    SQLite(const Path & path, bool create = true);
+    SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal);
     SQLite(const SQLite & from) = delete;
     SQLite& operator = (const SQLite & from) = delete;
     SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index c4642d333..7c4112d32 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -221,6 +221,13 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
             Allow parsing of timestamps in builtins.fromTOML.
         )",
     },
+    {
+        .tag = Xp::ReadOnlyLocalStore,
+        .name = "read-only-local-store",
+        .description = R"(
+            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+        )",
+    },
 }};
 
 static_assert(
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 892c6c371..507b0cc06 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -31,6 +31,7 @@ enum struct ExperimentalFeature
     DaemonTrustOverride,
     DynamicDerivations,
     ParseTomlTimestamps,
+    ReadOnlyLocalStore,
 };
 
 /**
diff --git a/tests/local.mk b/tests/local.mk
index 8e387fe45..88848926b 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -136,7 +136,8 @@ nix_tests = \
   impure-derivations.sh \
   path-from-hash-part.sh \
   test-libstoreconsumer.sh \
-  toString-path.sh
+  toString-path.sh \
+  read-only-store.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh
diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh
new file mode 100644
index 000000000..d63920c19
--- /dev/null
+++ b/tests/read-only-store.sh
@@ -0,0 +1,42 @@
+source common.sh
+
+enableFeatures "read-only-local-store"
+
+needLocalStore "cannot open store read-only when daemon has already opened it writeable"
+
+clearStore
+
+happy () {
+    # We can do a read-only query just fine with a read-only store
+    nix --store local?read-only=true path-info $dummyPath
+    
+    # We can "write" an already-present store-path a read-only store, because no IO is actually required
+    nix-store --store local?read-only=true --add dummy
+}
+## Testing read-only mode without forcing the underlying store to actually be read-only
+
+# Make sure the command fails when the store doesn't already have a database
+expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode"
+
+# Make sure the store actually has a current-database, with at least one store object
+dummyPath=$(nix-store --add dummy)
+
+# Try again and make sure we fail when adding a item not already in the store
+expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database"
+
+# Test a few operations that should work with the read-only store in its current state
+happy
+
+## Testing read-only mode with an underlying store that is actually read-only
+
+# Ensure store is actually read-only
+chmod -R -w $TEST_ROOT/store
+chmod -R -w $TEST_ROOT/var
+
+# Make sure we fail on add operations on the read-only store
+# This is only for adding files that are not *already* in the store
+expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'"
+expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied"
+
+# Test the same operations from before should again succeed
+happy

From f2b54e3b71891c29f951e46b00c6edc4807b3b63 Mon Sep 17 00:00:00 2001
From: fricklerhandwerk <valentin@fricklerhandwerk.de>
Date: Sat, 17 Jun 2023 03:25:57 +0200
Subject: [PATCH 127/183] add links to environment variables documentation

---
 .../src/command-ref/conf-file-prefix.md       | 25 ++++++++-----------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index 44b7ba86d..d2989ff20 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -6,28 +6,25 @@
 
 By default Nix reads settings from the following places:
 
-  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e.
-    `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if
-    `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded
-    to the Nix daemon. The client assumes that the daemon has already
-    loaded them.
+  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set.
 
-  - If `NIX_USER_CONF_FILES` is set, then each path separated by `:`
-    will be loaded in reverse order.
+    Values loaded in this file are not forwarded to the Nix daemon.
+    The client assumes that the daemon has already loaded them.
 
-    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
-    and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to
-    `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config`
-    as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+  - If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order.
 
-  - If `NIX_CONFIG` is set, its contents is treated as the contents of
-    a configuration file.
+    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME).
+    If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+
+  - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file.
 
 The configuration files consist of `name = value` pairs, one per
 line. Other files can be included with a line like `include path`,
 where *path* is interpreted relative to the current conf file and a
 missing file is an error unless `!include` is used instead. Comments
-start with a `#` character. Here is an example configuration file:
+start with a `#` character.
+
+Here is an example configuration file:
 
     keep-outputs = true       # Nice for developers
     keep-derivations = true   # Idem

From 38bd1cc9bc80e6dec59e56a7d1ce19379915a910 Mon Sep 17 00:00:00 2001
From: fricklerhandwerk <valentin@fricklerhandwerk.de>
Date: Sat, 17 Jun 2023 03:53:37 +0200
Subject: [PATCH 128/183] split configuration file page into sections

this makes it easier to scan for specific information, such as the
format of command line flags
---
 .../src/command-ref/conf-file-prefix.md       | 53 +++++++++++--------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index d2989ff20..29325ee45 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -4,6 +4,10 @@
 
 # Description
 
+Nix supports a variety of configuration settings, which are read from configuration files or taken as command line flags.
+
+## Configuration file
+
 By default Nix reads settings from the following places:
 
   - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set.
@@ -18,32 +22,39 @@ By default Nix reads settings from the following places:
 
   - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file.
 
-The configuration files consist of `name = value` pairs, one per
-line. Other files can be included with a line like `include path`,
-where *path* is interpreted relative to the current conf file and a
-missing file is an error unless `!include` is used instead. Comments
-start with a `#` character.
+### File format
 
-Here is an example configuration file:
+Configuration files consist of `name = value` pairs, one per line.
+Comments start with a `#` character.
 
-    keep-outputs = true       # Nice for developers
-    keep-derivations = true   # Idem
+Example:
 
-You can override settings on the command line using the `--option`
-flag, e.g. `--option keep-outputs false`. Every configuration setting
-also has a corresponding command line flag, e.g. `--max-jobs 16`; for
-Boolean settings, there are two flags to enable or disable the setting
-(e.g. `--keep-failed` and `--no-keep-failed`).
+```
+keep-outputs = true       # Nice for developers
+keep-derivations = true   # Idem
+```
 
-A configuration setting usually overrides any previous value. However,
-you can prefix the name of the setting by `extra-` to *append* to the
-previous value. For instance,
+Other files can be included with a line like `include <path>`, where `<path>` is interpreted relative to the current configuration file.
+A missing file is an error unless `!include` is used instead.
 
-    substituters = a b
-    extra-substituters = c d
+A configuration setting usually overrides any previous value.
+However, you can prefix the name of the setting by `extra-` to *append* to the previous value.
 
-defines the `substituters` setting to be `a b c d`. This is also
-available as a command line flag (e.g. `--extra-substituters`).
+For instance,
 
-The following settings are currently available:
+```
+substituters = a b
+extra-substituters = c d
+```
+
+defines the `substituters` setting to be `a b c d`.
+
+## Command line flags
+
+Every configuration setting has a corresponding command line flag (e.g. `--max-jobs 16`).
+Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`).
+
+Existing settings can be appended to using the `extra-` prefix (e.g. `--extra-substituters`).
+
+# Available settings
 

From bc7324e912578cdbeb1f10bf1c8d9b447a416dbd Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 20 Jun 2023 12:12:03 +0200
Subject: [PATCH 129/183] clarify read order for configuration settings

---
 doc/manual/src/command-ref/conf-file-prefix.md | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index 29325ee45..3cd622247 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -8,19 +8,19 @@ Nix supports a variety of configuration settings, which are read from configurat
 
 ## Configuration file
 
-By default Nix reads settings from the following places:
+By default Nix reads settings from the following places, in that order:
 
-  - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set.
+1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set.
 
-    Values loaded in this file are not forwarded to the Nix daemon.
-    The client assumes that the daemon has already loaded them.
+   Values loaded in this file are not forwarded to the Nix daemon.
+   The client assumes that the daemon has already loaded them.
 
-  - If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order.
+1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order.
 
-    Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME).
-    If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+   Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME).
+   If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
 
-  - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file.
+1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file.
 
 ### File format
 
@@ -38,7 +38,7 @@ Other files can be included with a line like `include <path>`, where `<path>` is
 A missing file is an error unless `!include` is used instead.
 
 A configuration setting usually overrides any previous value.
-However, you can prefix the name of the setting by `extra-` to *append* to the previous value.
+However, for settings that take a list of items, you can prefix the name of the setting by `extra-` to *append* to the previous value.
 
 For instance,
 

From 68c62193439ec90d34a7988c6a1a73b3e553a435 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 20 Jun 2023 12:12:23 +0200
Subject: [PATCH 130/183] clarify setting options on the command line

---
 doc/manual/src/command-ref/conf-file-prefix.md | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md
index 3cd622247..1e4085977 100644
--- a/doc/manual/src/command-ref/conf-file-prefix.md
+++ b/doc/manual/src/command-ref/conf-file-prefix.md
@@ -49,12 +49,22 @@ extra-substituters = c d
 
 defines the `substituters` setting to be `a b c d`.
 
+Unknown option names are not an error, and are simply ignored with a warning.
+
 ## Command line flags
 
-Every configuration setting has a corresponding command line flag (e.g. `--max-jobs 16`).
-Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`).
+Configuration options can be set on the command line, overriding the values set in the [configuration file](#configuration-file):
 
-Existing settings can be appended to using the `extra-` prefix (e.g. `--extra-substituters`).
+- Every configuration setting has corresponding command line flag (e.g. `--max-jobs 16`).
+  Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`).
+
+  Unknown option names are invalid flags (unless there is already a flag with that name), and are rejected with an error.
+
+- The flag `--option <name> <value>` is interpreted exactly like a `<name> = <value>` in a setting file.
+
+  Unknown option names are ignored with a warning.
+
+The `extra-` prefix is supported for settings that take a list of items (e.g. `--extra-trusted users alice` or `--option extra-trusted-users alice`).
 
 # Available settings
 

From e91d19db5f827cec56e32fe9b7c07c8d2c546ce6 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 20 Jun 2023 12:23:53 +0200
Subject: [PATCH 131/183] be more serious about security risks with trusted
 users

---
 src/nix/daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 6dcd7f94d..41f826228 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -64,7 +64,7 @@ struct AuthorizationSettings : Config {
           > **Warning**
           >
           > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system.
-          > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them.
+          > For example, the user can access or replace store path contents that are critical for system security.
         )"};
 
     /**

From 71317162c50df91d421eacdf87d50b4ba6d37635 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 20 Jun 2023 13:44:04 +0200
Subject: [PATCH 132/183] use more self-descriptive section headings

---
 doc/manual/src/contributing/hacking.md | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index c57d45138..12b91c79e 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -12,14 +12,15 @@ The following instructions assume you already have some version of Nix installed
 
 [installation instructions]: ../installation/installation.md
 
-## Nix with flakes
+## Building Nix with flakes
 
-This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes.
+This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled.
+See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces.
 
-[flakes]: ../command-ref/new-cli/nix3-flake.md#description
+[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes
+[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command
 
-To build all dependencies and start a shell in which all environment
-variables are set up so that those dependencies can be found:
+To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found:
 
 ```console
 $ nix develop
@@ -55,7 +56,7 @@ To install it in `$(pwd)/outputs` and test it:
 nix (Nix) 2.12
 ```
 
-To build a release version of Nix:
+To build a release version of Nix for the current operating system and CPU architecture:
 
 ```console
 $ nix build
@@ -63,12 +64,9 @@ $ nix build
 
 You can also build Nix for one of the [supported target platforms](#target-platforms).
 
-## Classic Nix
+## Building Nix
 
-This section is for Nix without [flakes].
-
-To build all dependencies and start a shell in which all environment
-variables are set up so that those dependencies can be found:
+To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found:
 
 ```console
 $ nix-shell
@@ -102,7 +100,7 @@ To install it in `$(pwd)/outputs` and test it:
 nix (Nix) 2.12
 ```
 
-To build Nix for the current operating system and CPU architecture use
+To build a release version of Nix for the current operating system and CPU architecture:
 
 ```console
 $ nix-build

From 3a20c7c46c64d2feb8caba767724a78d7efa3279 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Tue, 20 Jun 2023 22:51:29 +0200
Subject: [PATCH 133/183] Update tests/linux-sandbox.sh

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 tests/linux-sandbox.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 19a7527bf..f2a77174d 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -52,7 +52,6 @@ testCert () {
     certFile=$3    # a string that can be the path to a cert file
     [ "$mode" == fixed-output ] && ret=1 || ret=100
     expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" |
-        # tee /dev/stderr |
         grepQuiet "CERT_${expectation}_IN_SANDBOX"
 }
 

From a78f929065fd209e3d89853c6beb48942e3820e9 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 20 Jun 2023 12:39:07 +0200
Subject: [PATCH 134/183] fix anchor link

---
 doc/manual/src/contributing/hacking.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 12b91c79e..310d69310 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi
 $ nix build
 ```
 
-You can also build Nix for one of the [supported target platforms](#target-platforms).
+You can also build Nix for one of the [supported target platforms](#platforms).
 
 ## Building Nix
 
@@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi
 $ nix-build
 ```
 
-You can also build Nix for one of the [supported target platforms](#target-platforms).
+You can also build Nix for one of the [supported target platforms](#platforms).
 
 ## Platforms
 

From 085104944b2957f4269fcbfaf0d1874ab0c9ce84 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 21 Jun 2023 09:46:23 +0200
Subject: [PATCH 135/183] add redirects to changed anchors

---
 doc/manual/redirects.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index 5cd6fdea2..6813dc6d6 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -342,6 +342,10 @@ const redirects = {
   "installation/installing-binary.html": {
     "uninstalling": "uninstall.html"
   }
+  "contributing/hacking.html": {
+    "nix-with-flakes": "#building-nix-with-flakes"
+    "classic-nix": "#building-nix"
+  }
 };
 
 // the following code matches the current page's URL against the set of redirects.

From 5f9a921bc1e6d3e6bf65601be565585cf0190e8f Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 21 Jun 2023 14:31:09 +0200
Subject: [PATCH 136/183] do not use "target", as it's a loaded term in the
 domain of compilers

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/contributing/hacking.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 310d69310..6c6f2eb52 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi
 $ nix build
 ```
 
-You can also build Nix for one of the [supported target platforms](#platforms).
+You can also build Nix for one of the [supported platforms](#platforms).
 
 ## Building Nix
 
@@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi
 $ nix-build
 ```
 
-You can also build Nix for one of the [supported target platforms](#platforms).
+You can also build Nix for one of the [supported platforms](#platforms).
 
 ## Platforms
 

From 5cc22e337023cee9cbffd3f9d2ff0eb3f18a4d12 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 15 Jun 2023 22:29:03 -0400
Subject: [PATCH 137/183] Clarify docs on deleting generations, including
 fixing a mistake

Deleting store info corrected (there is a foot-gun in Nix with
`--delete-generations old`!)

Also a few things are cleaned up based on feedback.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 .../src/command-ref/nix-collect-garbage.md    |  2 +-
 .../command-ref/nix-env/delete-generations.md | 28 ++++++++++++-------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md
index a679ceaf7..3cab79f0e 100644
--- a/doc/manual/src/command-ref/nix-collect-garbage.md
+++ b/doc/manual/src/command-ref/nix-collect-garbage.md
@@ -57,7 +57,7 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto
   Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time).
   *period* is a value such as `30d`, which would mean 30 days.
 
-  This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile.
+  This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile.
   See the documentation of that command for additional information about the *period* argument.
 
 {{#include ./opt-common.md}}
diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md
index d828a5b9e..adc6fc219 100644
--- a/doc/manual/src/command-ref/nix-env/delete-generations.md
+++ b/doc/manual/src/command-ref/nix-env/delete-generations.md
@@ -20,22 +20,30 @@ This operation deletes the specified generations of the current profile.
 
 - The special value <span id="generations-old">`old`</span>
 
-  Delete all generations older than the current one.
+  Delete all generations except the current one.
 
-- <span id="generations-days">`<days>d`</span>:\
-  The last *days* days
+  > **WARNING**
+  >
+  > Older *and newer* generations will be deleted by this operation.
+  >
+  > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest.
+  > Because one can roll back to a previous generation, it is possible to have generations newer than the current one.
+  > They will also be deleted.
+
+- <span id="generations-time">`<number>d`</span>:\
+  The last *number* days
 
   *Example*: `30d`
 
-  Delete all generations older than *days* days.
-  The generation that was active at that point in time is excluded, and will not be deleted.
+  Delete all generations created more than *number* days ago, except the most recent one of them.
+  This allows rolling back to generations that were available within the specified period.
 
-- <span id="generations-count">`+<count>`</span>:\
-  The last *count* generations up to the present
+- <span id="generations-count">`+<number>`</span>:\
+  The last *number* generations up to the present
 
   *Example*: `+5`
 
-  Keep the last *count* generations, along with any newer than current.
+  Keep the last *number* generations, along with any newer than current.
 
 Periodically deleting old generations is important to make garbage collection
 effective.
@@ -61,7 +69,7 @@ $ nix-env --delete-generations 3 4 8
 
 Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those.
 
-## Keep most-recent by count count
+## Keep most-recent by count (number of generations)
 
 ```console
 $ nix-env --delete-generations +5
@@ -72,7 +80,7 @@ Suppose `30` is the current generation, and we currently have generations number
 Then this command will delete generations `20` through `25` (`<= 30 - 5`),
 and keep generations `26` through `31` (`> 30 - 5`).
 
-## Keep most-recent in days
+## Keep most-recent by time (number of days)
 
 ```console
 $ nix-env --delete-generations 30d

From c48277c1c15f0b6be6a47271a6f8399fad96f7fd Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Thu, 22 Jun 2023 18:19:42 +0800
Subject: [PATCH 138/183] libexpr: add tests for `nix::Value::print`

---
 src/libexpr/tests/value/print.cc | 170 +++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 src/libexpr/tests/value/print.cc

diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc
new file mode 100644
index 000000000..2268658b2
--- /dev/null
+++ b/src/libexpr/tests/value/print.cc
@@ -0,0 +1,170 @@
+#include "tests/libexpr.hh"
+
+#include "value.hh"
+
+namespace nix {
+
+using namespace testing;
+
+struct ValuePrintingTests : LibExprTest
+{
+    template<class... A>
+    void test(Value v, std::string_view expected, A... args)
+    {
+        std::stringstream out;
+        v.print(state.symbols, out, args...);
+        ASSERT_EQ(out.str(), expected);
+    }
+};
+
+TEST_F(ValuePrintingTests, tInt)
+{
+    Value vInt;
+    vInt.mkInt(10);
+    test(vInt, "10");
+}
+
+TEST_F(ValuePrintingTests, tBool)
+{
+    Value vBool;
+    vBool.mkBool(true);
+    test(vBool, "true");
+}
+
+TEST_F(ValuePrintingTests, tString)
+{
+    Value vString;
+    vString.mkString("some-string");
+    test(vString, "\"some-string\"");
+}
+
+TEST_F(ValuePrintingTests, tPath)
+{
+    Value vPath;
+    vPath.mkString("/foo");
+    test(vPath, "\"/foo\"");
+}
+
+TEST_F(ValuePrintingTests, tNull)
+{
+    Value vNull;
+    vNull.mkNull();
+    test(vNull, "null");
+}
+
+TEST_F(ValuePrintingTests, tAttrs)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("one"), &vOne);
+    builder.insert(state.symbols.create("two"), &vTwo);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs, "{ one = 1; two = 2; }");
+}
+
+TEST_F(ValuePrintingTests, tList)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    Value vList;
+    state.mkList(vList, 5);
+    vList.bigList.elems[0] = &vOne;
+    vList.bigList.elems[1] = &vTwo;
+    vList.bigList.size = 3;
+
+    test(vList, "[ 1 2 (nullptr) ]");
+}
+
+TEST_F(ValuePrintingTests, vThunk)
+{
+    Value vThunk;
+    vThunk.mkThunk(nullptr, nullptr);
+
+    test(vThunk, "<CODE>");
+}
+
+TEST_F(ValuePrintingTests, vApp)
+{
+    Value vApp;
+    vApp.mkApp(nullptr, nullptr);
+
+    test(vApp, "<CODE>");
+}
+
+TEST_F(ValuePrintingTests, vLambda)
+{
+    Value vLambda;
+    vLambda.mkLambda(nullptr, nullptr);
+
+    test(vLambda, "<LAMBDA>");
+}
+
+TEST_F(ValuePrintingTests, vPrimOp)
+{
+    Value vPrimOp;
+    vPrimOp.mkPrimOp(nullptr);
+
+    test(vPrimOp, "<PRIMOP>");
+}
+
+TEST_F(ValuePrintingTests, vPrimOpApp)
+{
+    Value vPrimOpApp;
+    vPrimOpApp.mkPrimOpApp(nullptr, nullptr);
+
+    test(vPrimOpApp, "<PRIMOP-APP>");
+}
+
+TEST_F(ValuePrintingTests, vExternal)
+{
+    struct MyExternal : ExternalValueBase
+    {
+    public:
+        std::string showType() const override
+        {
+            return "";
+        }
+        std::string typeOf() const override
+        {
+            return "";
+        }
+        virtual std::ostream & print(std::ostream & str) const override
+        {
+            str << "testing-external!";
+            return str;
+        }
+    } myExternal;
+    Value vExternal;
+    vExternal.mkExternal(&myExternal);
+
+    test(vExternal, "testing-external!");
+}
+
+TEST_F(ValuePrintingTests, vFloat)
+{
+    Value vFloat;
+    vFloat.mkFloat(2.0);
+
+    test(vFloat, "2");
+}
+
+TEST_F(ValuePrintingTests, vBlackhole)
+{
+    Value vBlackhole;
+    vBlackhole.mkBlackhole();
+    test(vBlackhole, "«potential infinite recursion»");
+}
+
+} // namespace nix

From 1400fde144e1fdde2647fda5a7f9511f25572a35 Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Thu, 22 Jun 2023 12:27:19 +0800
Subject: [PATCH 139/183] libexpr: extend `Value::print` to allow limited depth

---
 src/libexpr/eval.cc              | 20 ++++++----
 src/libexpr/tests/value/print.cc | 66 ++++++++++++++++++++++++++++++++
 src/libexpr/value.hh             |  5 ++-
 3 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 71fd6e6e4..da55d3f9e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -95,11 +95,16 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-void Value::print(const SymbolTable & symbols, std::ostream & str,
-    std::set<const void *> * seen) const
+void Value::print(const SymbolTable &symbols, std::ostream &str,
+                  std::set<const void *> *seen, int depth) const
+
 {
     checkInterrupt();
 
+    if (depth <= 0) {
+        str << "«too deep»";
+        return;
+    }
     switch (internalType) {
     case tInt:
         str << integer;
@@ -123,7 +128,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
             str << "{ ";
             for (auto & i : attrs->lexicographicOrder(symbols)) {
                 str << symbols[i->name] << " = ";
-                i->value->print(symbols, str, seen);
+                i->value->print(symbols, str, seen, depth - 1);
                 str << "; ";
             }
             str << "}";
@@ -139,7 +144,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
             str << "[ ";
             for (auto v2 : listItems()) {
                 if (v2)
-                    v2->print(symbols, str, seen);
+                    v2->print(symbols, str, seen, depth - 1);
                 else
                     str << "(nullptr)";
                 str << " ";
@@ -181,11 +186,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
     }
 }
 
-
-void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const
-{
+void Value::print(const SymbolTable &symbols, std::ostream &str,
+                  bool showRepeated, int depth) const {
     std::set<const void *> seen;
-    print(symbols, str, showRepeated ? nullptr : &seen);
+    print(symbols, str, showRepeated ? nullptr : &seen, depth);
 }
 
 // Pretty print types for assertion errors
diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc
index 2268658b2..5e96e12ec 100644
--- a/src/libexpr/tests/value/print.cc
+++ b/src/libexpr/tests/value/print.cc
@@ -167,4 +167,70 @@ TEST_F(ValuePrintingTests, vBlackhole)
     test(vBlackhole, "«potential infinite recursion»");
 }
 
+TEST_F(ValuePrintingTests, depthAttrs)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("one"), &vOne);
+    builder.insert(state.symbols.create("two"), &vTwo);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    BindingsBuilder builder2(state, state.allocBindings(10));
+    builder2.insert(state.symbols.create("one"), &vOne);
+    builder2.insert(state.symbols.create("two"), &vTwo);
+    builder2.insert(state.symbols.create("nested"), &vAttrs);
+
+    Value vNested;
+    vNested.mkAttrs(builder2.finish());
+
+    test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1);
+    test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2);
+    test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3);
+    test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4);
+}
+
+TEST_F(ValuePrintingTests, depthList)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("one"), &vOne);
+    builder.insert(state.symbols.create("two"), &vTwo);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    BindingsBuilder builder2(state, state.allocBindings(10));
+    builder2.insert(state.symbols.create("one"), &vOne);
+    builder2.insert(state.symbols.create("two"), &vTwo);
+    builder2.insert(state.symbols.create("nested"), &vAttrs);
+
+    Value vNested;
+    vNested.mkAttrs(builder2.finish());
+
+    Value vList;
+    state.mkList(vList, 5);
+    vList.bigList.elems[0] = &vOne;
+    vList.bigList.elems[1] = &vTwo;
+    vList.bigList.elems[2] = &vNested;
+    vList.bigList.size = 3;
+
+    test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1);
+    test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2);
+    test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3);
+    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4);
+    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5);
+}
+
 } // namespace nix
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 89c0c36fd..6b32e2b04 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include <cassert>
+#include <climits>
 
 #include "symbol-table.hh"
 #include "value/context.hh"
@@ -137,11 +138,11 @@ private:
 
     friend std::string showType(const Value & v);
 
-    void print(const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen) const;
+    void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const;
 
 public:
 
-    void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const;
+    void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const;
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's

From 97df060588e4d328a7e219107553087149a77bac Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 22 Jun 2023 14:23:25 -0400
Subject: [PATCH 140/183] Better document build failure exit codes

- Improved API docs from comment

- Exit codes are for `nix-build`, not just `nix-store --release`

- Make note in tests so the magic numbers are not surprising

Picking up where #8387 left off.
---
 doc/manual/src/command-ref/nix-build.md       |  2 ++
 .../src/command-ref/nix-store/realise.md      | 31 +----------------
 .../src/command-ref/status-build-failure.md   | 34 +++++++++++++++++++
 src/libstore/build/entry-points.cc            | 10 +++---
 src/libstore/build/worker.cc                  | 11 ++----
 src/libstore/build/worker.hh                  | 23 ++++++++++++-
 tests/check.sh                                |  3 ++
 tests/linux-sandbox.sh                        |  3 ++
 8 files changed, 72 insertions(+), 45 deletions(-)
 create mode 100644 doc/manual/src/command-ref/status-build-failure.md

diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md
index 7369ea2eb..b548edf82 100644
--- a/doc/manual/src/command-ref/nix-build.md
+++ b/doc/manual/src/command-ref/nix-build.md
@@ -70,6 +70,8 @@ except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](n
     Change the name of the symlink to the output path created from
     `result` to *outlink*.
 
+{{#include ./status-build-failure.md}}
+
 {{#include ./opt-common.md}}
 
 {{#include ./env-common.md}}
diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md
index 6b50d2145..c19aea75e 100644
--- a/doc/manual/src/command-ref/nix-store/realise.md
+++ b/doc/manual/src/command-ref/nix-store/realise.md
@@ -54,36 +54,7 @@ The following flags are available:
     previous build, the new output path is left in
     `/nix/store/name.check.`
 
-Special exit codes:
-
-  - `100`\
-    Generic build failure, the builder process returned with a non-zero
-    exit code.
-
-  - `101`\
-    Build timeout, the build was aborted because it did not complete
-    within the specified `timeout`.
-
-  - `102`\
-    Hash mismatch, the build output was rejected because it does not
-    match the [`outputHash` attribute of the
-    derivation](@docroot@/language/advanced-attributes.md).
-
-  - `104`\
-    Not deterministic, the build succeeded in check mode but the
-    resulting output is not binary reproducible.
-
-With the `--keep-going` flag it's possible for multiple failures to
-occur, in this case the 1xx status codes are or combined using binary
-or.
-
-    1100100
-       ^^^^
-       |||`- timeout
-       ||`-- output hash mismatch
-       |`--- build failure
-       `---- not deterministic
-
+{{#include ../status-build-failure.md}}
 
 {{#include ./opt-common.md}}
 
diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/src/command-ref/status-build-failure.md
new file mode 100644
index 000000000..06114eb29
--- /dev/null
+++ b/doc/manual/src/command-ref/status-build-failure.md
@@ -0,0 +1,34 @@
+# Special exit codes for build failure
+
+1xx status codes are used when requested builds failed.
+The following codes are in use:
+
+- `100` Generic build failure
+
+  The builder process returned with a non-zero exit code.
+
+- `101` Build timeout
+
+  The build was aborted because it did not complete within the specified `timeout`.
+
+- `102` Hash mismatch
+
+  The build output was rejected because it does not match the
+  [`outputHash` attribute of the derivation](@docroot@/language/advanced-attributes.md).
+
+- `104` Not deterministic
+
+  The build succeeded in check mode but the resulting output is not binary reproducible.
+
+With the `--keep-going` flag it's possible for multiple failures to occur.
+In this case the 1xx status codes are or combined using
+[bitwise OR](https://en.wikipedia.org/wiki/Bitwise_operation#OR).
+
+```
+0b1100100
+     ^^^^
+     |||`- timeout
+     ||`-- output hash mismatch
+     |`--- build failure
+     `---- not deterministic
+```
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index edd6cb6d2..4aa4d6dca 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -31,11 +31,11 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     }
 
     if (failed.size() == 1 && ex) {
-        ex->status = worker.exitStatus();
+        ex->status = worker.failingExitStatus();
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
+        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
     }
 }
 
@@ -102,10 +102,10 @@ void Store::ensurePath(const StorePath & path)
 
     if (goal->exitCode != Goal::ecSuccess) {
         if (goal->ex) {
-            goal->ex->status = worker.exitStatus();
+            goal->ex->status = worker.failingExitStatus();
             throw std::move(*goal->ex);
         } else
-            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
+            throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
     }
 }
 
@@ -128,7 +128,7 @@ void Store::repairPath(const StorePath & path)
             goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
             worker.run(goals);
         } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
+            throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
     }
 }
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index ee334d54a..a9ca9cbbc 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -468,16 +468,9 @@ void Worker::waitForInput()
 }
 
 
-unsigned int Worker::exitStatus()
+unsigned int Worker::failingExitStatus()
 {
-    /*
-     * 1100100
-     *    ^^^^
-     *    |||`- timeout
-     *    ||`-- output hash mismatch
-     *    |`--- build failure
-     *    `---- not deterministic
-     */
+    // See API docs in header for explanation
     unsigned int mask = 0;
     bool buildFailure = permanentFailure || timedOut || hashMismatch;
     if (buildFailure)
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 63624d910..5abceca0d 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -280,7 +280,28 @@ public:
      */
     void waitForInput();
 
-    unsigned int exitStatus();
+    /***
+     * The exit status in case of failure.
+     *
+     * In the case of a build failure, returned value follows this
+     * bitmask:
+     *
+     * ```
+     * 0b1100100
+     *      ^^^^
+     *      |||`- timeout
+     *      ||`-- output hash mismatch
+     *      |`--- build failure
+     *      `---- not deterministic
+     * ```
+     *
+     * In other words, the failure code is at least 100 (0b1100100), but
+     * might also be greater.
+     *
+     * Otherwise (no build failure, but some other sort of failure by
+     * assumption), this returned value is 1.
+     */
+    unsigned int failingExitStatus();
 
     /**
      * Check whether the given valid path exists and has the right
diff --git a/tests/check.sh b/tests/check.sh
index 645b90222..e13abf747 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -18,6 +18,9 @@ clearStore
 nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
+# Build failure exit codes (100, 104, etc.) are from
+# doc/manual/src/command-ref/status-build-failure.md
+
 # check for dangling temporary build directories
 # only retain if build fails and --keep-failed is specified, or...
 # ...build is non-deterministic and --check and --keep-failed are both specified
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index f2a77174d..ff7d257bd 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -36,11 +36,13 @@ nix-sandbox-build dependencies.nix --check
 # Test that sandboxed builds with --check and -K can move .check directory to store
 nix-sandbox-build check.nix -A nondeterministic
 
+# `100 + 4` means non-determinstic, see doc/manual/src/command-ref/status-build-failure.md
 expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log
 grepQuietInverse 'error: renaming' $TEST_ROOT/log
 grepQuiet 'may not be deterministic' $TEST_ROOT/log
 
 # Test that sandboxed builds cannot write to /etc easily
+# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md
 expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' |
     grepQuiet "/etc/test: Permission denied"
 
@@ -50,6 +52,7 @@ testCert () {
     expectation=$1 # "missing" | "present"
     mode=$2        # "normal" | "fixed-output"
     certFile=$3    # a string that can be the path to a cert file
+    # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md
     [ "$mode" == fixed-output ] && ret=1 || ret=100
     expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" |
         grepQuiet "CERT_${expectation}_IN_SANDBOX"

From 9d8c4ac446bed82eb22a989cd81982acd722c58a Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Fri, 23 Jun 2023 13:35:41 +0800
Subject: [PATCH 141/183] libexpr: remove unused token `ATTRPATH` in token
 declaration

---
 src/libexpr/parser.y | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 3b545fd84..5c9597f8a 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -330,7 +330,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %type <ind_string_parts> ind_string_parts
 %type <e> path_start string_parts string_attr
 %type <id> attr
-%token <id> ID ATTRPATH
+%token <id> ID
 %token <str> STR IND_STR
 %token <n> INT
 %token <nf> FLOAT

From 484290a9e05ea840f9bc6ba3cc98d64ccffe202b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 23 Jun 2023 12:01:10 -0400
Subject: [PATCH 142/183] Use a struct not `std::pair` for `SearchPathElem`

I got very confused trying to keep all the `first` and `second` straight
reading the code, *especially* as there is also another `(boolean,
string)` pair type also being used.

Named fields is much better.

There are other cleanups that we can do (for example, the existing
TODO), but we can do them later. Doing them now would just make this
harder to review.
---
 src/libexpr/eval.hh    |  8 ++++++--
 src/libexpr/parser.y   | 33 ++++++++++++++++++---------------
 src/libexpr/primops.cc |  9 ++++++---
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 8e41bdbd0..7b726a78f 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -65,8 +65,12 @@ std::string printValue(const EvalState & state, const Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
-// FIXME: maybe change this to an std::variant<SourcePath, URL>.
-typedef std::pair<std::string, std::string> SearchPathElem;
+struct SearchPathElem
+{
+    std::string prefix;
+    // FIXME: maybe change this to an std::variant<SourcePath, URL>.
+    std::string path;
+};
 typedef std::list<SearchPathElem> SearchPath;
 
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 5c9597f8a..03e6fb02c 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -741,7 +741,10 @@ void EvalState::addToSearchPath(const std::string & s)
         path = std::string(s, pos + 1);
     }
 
-    searchPath.emplace_back(prefix, path);
+    searchPath.emplace_back(SearchPathElem {
+        .prefix = prefix,
+        .path = path,
+    });
 }
 
 
@@ -755,11 +758,11 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
 {
     for (auto & i : searchPath) {
         std::string suffix;
-        if (i.first.empty())
+        if (i.prefix.empty())
             suffix = concatStrings("/", path);
         else {
-            auto s = i.first.size();
-            if (path.compare(0, s, i.first) != 0 ||
+            auto s = i.prefix.size();
+            if (path.compare(0, s, i.prefix) != 0 ||
                 (path.size() > s && path[s] != '/'))
                 continue;
             suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
@@ -785,47 +788,47 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
 
 std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem)
 {
-    auto i = searchPathResolved.find(elem.second);
+    auto i = searchPathResolved.find(elem.path);
     if (i != searchPathResolved.end()) return i->second;
 
     std::pair<bool, std::string> res;
 
-    if (EvalSettings::isPseudoUrl(elem.second)) {
+    if (EvalSettings::isPseudoUrl(elem.path)) {
         try {
             auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath;
+                store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath;
             res = { true, store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path)
             });
             res = { false, "" };
         }
     }
 
-    else if (hasPrefix(elem.second, "flake:")) {
+    else if (hasPrefix(elem.path, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false);
-        debug("fetching flake search path element '%s''", elem.second);
+        auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", elem.path);
         auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
         res = { true, store->toRealPath(storePath) };
     }
 
     else {
-        auto path = absPath(elem.second);
+        auto path = absPath(elem.path);
         if (pathExists(path))
             res = { true, path };
         else {
             logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path)
             });
             res = { false, "" };
         }
     }
 
-    debug("resolved search path element '%s' to '%s'", elem.second, res.second);
+    debug("resolved search path element '%s' to '%s'", elem.path, res.second);
 
-    searchPathResolved[elem.second] = res;
+    searchPathResolved[elem.path] = res;
     return res;
 }
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5b2f7e8b7..9dd7bae02 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1656,7 +1656,10 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
             }));
         }
 
-        searchPath.emplace_back(prefix, path);
+        searchPath.emplace_back(SearchPathElem {
+            .prefix = prefix,
+            .path = path,
+        });
     }
 
     auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
@@ -4129,8 +4132,8 @@ void EvalState::createBaseEnv()
     int n = 0;
     for (auto & i : searchPath) {
         auto attrs = buildBindings(2);
-        attrs.alloc("path").mkString(i.second);
-        attrs.alloc("prefix").mkString(i.first);
+        attrs.alloc("path").mkString(i.path);
+        attrs.alloc("prefix").mkString(i.prefix);
         (v.listElems()[n++] = allocValue())->mkAttrs(attrs);
     }
     addConstant("__nixPath", v);

From a7b49086c76bc181ab1cf1d3c7fba44b06d2f1dd Mon Sep 17 00:00:00 2001
From: Michael Hoang <enzime@users.noreply.github.com>
Date: Thu, 14 Oct 2021 23:44:45 +1100
Subject: [PATCH 143/183] Add `dirtyRev` and `dirtyShortRev` to `fetchGit`

Fixes #4682
---
 src/libexpr/primops/fetchTree.cc | 7 ++++++-
 src/libfetchers/git.cc           | 9 ++++++++-
 src/nix/flake.cc                 | 6 ++++++
 tests/fetchGit.sh                | 6 ++++++
 tests/flakes/flakes.sh           | 5 +++++
 5 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index cd7039025..bfae34a48 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -22,7 +22,7 @@ void emitTreeAttrs(
 {
     assert(input.isLocked());
 
-    auto attrs = state.buildBindings(8);
+    auto attrs = state.buildBindings(10);
 
 
     state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
@@ -56,6 +56,11 @@ void emitTreeAttrs(
 
     }
 
+    if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
+        attrs.alloc("dirtyRev").mkString(*dirtyRev);
+        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
+    }
+
     if (auto lastModified = input.getLastModified()) {
         attrs.alloc("lastModified").mkInt(*lastModified);
         attrs.alloc("lastModifiedDate").mkString(
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 47282f6c4..be5842d53 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -243,6 +243,13 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
         "lastModified",
         workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
+    if (workdirInfo.hasHead) {
+        input.attrs.insert_or_assign("dirtyRev", chomp(
+            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
+        input.attrs.insert_or_assign("dirtyShortRev", chomp(
+            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
+    }
+
     return {std::move(storePath), input};
 }
 }  // end namespace
@@ -283,7 +290,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 1eea52e15..8d39de389 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -179,6 +179,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
             j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs());
             if (auto rev = flake.lockedRef.input.getRev())
                 j["revision"] = rev->to_string(Base16, false);
+            if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev"))
+                j["dirtyRevision"] = *dirtyRev;
             if (auto revCount = flake.lockedRef.input.getRevCount())
                 j["revCount"] = *revCount;
             if (auto lastModified = flake.lockedRef.input.getLastModified())
@@ -204,6 +206,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
                 logger->cout(
                     ANSI_BOLD "Revision:" ANSI_NORMAL "      %s",
                     rev->to_string(Base16, false));
+            if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev"))
+                logger->cout(
+                    ANSI_BOLD "Revision:" ANSI_NORMAL "      %s",
+                    *dirtyRev);
             if (auto revCount = flake.lockedRef.input.getRevCount())
                 logger->cout(
                     ANSI_BOLD "Revisions:" ANSI_NORMAL "     %s",
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index e2ccb0e97..418b4f63f 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -105,6 +105,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 [[ $(cat $path2/dir1/foo) = foo ]]
 
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]]
 
 # ... unless we're using an explicit ref or rev.
 path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
@@ -119,6 +121,10 @@ git -C $repo commit -m 'Bla3' -a
 path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
 [[ $path2 = $path4 ]]
 
+[[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]]
+[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]]
+[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]]
+
 status=0
 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
 [[ "$status" = "102" ]]
diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh
index f2e216435..128f759ea 100644
--- a/tests/flakes/flakes.sh
+++ b/tests/flakes/flakes.sh
@@ -95,11 +95,16 @@ json=$(nix flake metadata flake1 --json | jq .)
 [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
 hash1=$(echo "$json" | jq -r .revision)
 
+echo foo > $flake1Dir/foo
+git -C $flake1Dir add $flake1Dir/foo
+[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]]
+
 echo -n '# foo' >> $flake1Dir/flake.nix
 flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD)
 git -C $flake1Dir commit -a -m 'Foo'
 flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD)
 hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision)
+[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]]
 [[ $hash1 != $hash2 ]]
 
 # Test 'nix build' on a flake.

From 559fd7ffe7a8cb61b11bec081f14ce3c3ffb210d Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 27 Jun 2023 14:58:29 +0200
Subject: [PATCH 144/183] nix flake check: improve error message if overlay is
 not a lambda (#8582)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* nix flake check: improve error message if overlay is not a lambda

Suppose you have an overlay like this

    {
      inputs = { /* ... */ };
      outputs = { flake-utils, ... }: flake-utils.lib.eachDefaultSystem
        (system: {
          overlays.default = final: prev: {

          };
        });
    }

then `nix flake check` (correctly) fails because `overlays` are supposed
to have the structure `overlays.<name> = final: prev: exp`. However, the
error-message is a little bit counter-intuitive:

    error: overlay does not take an argument named 'final'

While one might guess where the error actually comes from because the
trace above says `… while checking the overlay 'overlays.x86_64-linux'`
this is still pretty confusing because it complains about an argument
not being named `final` even though that's evidently the case.

With this change, the error-message actually makes it clear what's
wrong:

    [ma27@carsten:~/Projects/nix/tmp]$ nix flake check --extra-experimental-features 'nix-command flakes' path:$(pwd)
    error:
           … while checking flake output 'overlays'

             at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5:

                3|   outputs = { ... }: {
                4|     overlays.x86_64-linux.snens = final: prev: {
                 |     ^
                5|       kek = throw "snens";

           … while checking the overlay 'overlays.x86_64-linux'

             at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5:

                3|   outputs = { ... }: {
                4|     overlays.x86_64-linux.snens = final: prev: {
                 |     ^
                5|       kek = throw "snens";

           error: overlay is not a lambda, but a set instead
---
 src/nix/flake.cc      |  6 ++++--
 tests/flakes/check.sh | 12 ++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 8d39de389..b5f5d0cac 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -386,8 +386,10 @@ struct CmdFlakeCheck : FlakeCommand
         auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) {
             try {
                 state->forceValue(v, pos);
-                if (!v.isLambda()
-                    || v.lambda.fun->hasFormals()
+                if (!v.isLambda()) {
+                    throw Error("overlay is not a function, but %s instead", showType(v));
+                }
+                if (v.lambda.fun->hasFormals()
                     || !argHasName(v.lambda.fun->arg, "final"))
                     throw Error("overlay does not take an argument named 'final'");
                 auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh
index 34b82c61c..0433e5335 100644
--- a/tests/flakes/check.sh
+++ b/tests/flakes/check.sh
@@ -25,6 +25,18 @@ EOF
 
 (! nix flake check $flakeDir)
 
+cat > $flakeDir/flake.nix <<EOF
+{
+  outputs = { self, ... }: {
+    overlays.x86_64-linux.foo = final: prev: {
+    };
+  };
+}
+EOF
+
+checkRes=$(nix flake check $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true)
+echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead"
+
 cat > $flakeDir/flake.nix <<EOF
 {
   outputs = { self }: {

From e8067daf0955c297f389c968dab3e927b395de07 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 21 Jun 2023 16:06:16 -0400
Subject: [PATCH 145/183] Generialize `showType`

---
 src/libexpr/eval.cc | 23 ++++++++++++-----------
 src/libexpr/eval.hh |  5 ++++-
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 71fd6e6e4..8a0ff4cce 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -211,20 +211,21 @@ const Value * getPrimOp(const Value &v) {
     return primOp;
 }
 
-std::string_view showType(ValueType type)
+std::string_view showType(ValueType type, bool withArticle)
 {
+    #define WA(a, w) withArticle ? a " " w : w
     switch (type) {
-        case nInt: return "an integer";
-        case nBool: return "a Boolean";
-        case nString: return "a string";
-        case nPath: return "a path";
+        case nInt: return WA("an", "integer");
+        case nBool: return WA("a", "Boolean");
+        case nString: return WA("a", "string");
+        case nPath: return WA("a", "path");
         case nNull: return "null";
-        case nAttrs: return "a set";
-        case nList: return "a list";
-        case nFunction: return "a function";
-        case nExternal: return "an external value";
-        case nFloat: return "a float";
-        case nThunk: return "a thunk";
+        case nAttrs: return WA("a", "set");
+        case nList: return WA("a", "list");
+        case nFunction: return WA("a", "function");
+        case nExternal: return WA("an", "external value");
+        case nFloat: return WA("a", "float");
+        case nThunk: return WA("a", "thunk");
     }
     abort();
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 7b726a78f..0c07ae081 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -704,8 +704,11 @@ struct DebugTraceStacker {
 
 /**
  * @return A string representing the type of the value `v`.
+ *
+ * @param withArticle Whether to begin with an english article, e.g. "an
+ * integer" vs "integer".
  */
-std::string_view showType(ValueType type);
+std::string_view showType(ValueType type, bool withArticle = true);
 std::string showType(const Value & v);
 
 /**

From 4da7c866181c699b43d3cd2fff0afd2505774be7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 27 Jun 2023 09:32:40 -0400
Subject: [PATCH 146/183] Switch example to a primop this is less ill-advised

Any primop will do for this, so might as well use one that isn't impure.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/language/builtin-constants.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md
index e6bc7e915..eb8b53bf0 100644
--- a/doc/manual/src/language/builtin-constants.md
+++ b/doc/manual/src/language/builtin-constants.md
@@ -9,7 +9,8 @@ These constants are built into the Nix language evaluator:
   Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
 
   ```nix
-  if builtins ? getEnv then builtins.getEnv "PATH" else ""
+  # if hasContext is not available, we assume `s` has a context
+  if builtins ? hasContext then builtins.hasContext s else true
   ```
 
 - [`builtins.currentSystem`]{#builtins-currentSystem} (string)

From d40f0e534d657e619634ba204fb8970f4a01fbc5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 27 Jun 2023 09:35:51 -0400
Subject: [PATCH 147/183] Don't say this when we still pollute the global scope

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 doc/manual/src/language/builtin-constants.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md
index eb8b53bf0..99520ad3f 100644
--- a/doc/manual/src/language/builtin-constants.md
+++ b/doc/manual/src/language/builtin-constants.md
@@ -4,7 +4,7 @@ These constants are built into the Nix language evaluator:
 
 - [`builtins`]{#builtins-builtins} (attribute set)
 
-  Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope.
+  Contains all the [built-in functions](./builtins.md) and values.
 
   Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
 

From 22b278e011ab9c1328749a126514c57b89a39173 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 13 May 2023 13:52:45 -0400
Subject: [PATCH 148/183] Automatically document builtin constants

This is done in roughly the same way builtin functions are documented.

Also auto-link experimental features for primops, subsuming PR #8371.

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .gitignore                                    |   3 +-
 doc/manual/generate-builtin-constants.nix     |  29 ++
 doc/manual/generate-builtins.nix              |  18 +-
 doc/manual/local.mk                           |  16 +-
 .../src/language/builtin-constants-prefix.md  |   5 +
 .../src/language/builtin-constants-suffix.md  |   1 +
 doc/manual/src/language/builtin-constants.md  |  44 ---
 src/libexpr/eval.cc                           |  37 ++-
 src/libexpr/eval.hh                           |  87 +++++-
 src/libexpr/flake/flake.cc                    |   3 -
 src/libexpr/primops.cc                        | 283 +++++++++++++++---
 src/libexpr/primops.hh                        |  14 +-
 src/libexpr/primops/fetchClosure.cc           |   3 -
 src/libexpr/value.hh                          |  10 +-
 src/nix/main.cc                               |  41 ++-
 15 files changed, 447 insertions(+), 147 deletions(-)
 create mode 100644 doc/manual/generate-builtin-constants.nix
 create mode 100644 doc/manual/src/language/builtin-constants-prefix.md
 create mode 100644 doc/manual/src/language/builtin-constants-suffix.md
 delete mode 100644 doc/manual/src/language/builtin-constants.md

diff --git a/.gitignore b/.gitignore
index 29d9106ae..969194650 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,7 @@ perl/Makefile.config
 /doc/manual/generated/*
 /doc/manual/nix.json
 /doc/manual/conf-file.json
-/doc/manual/builtins.json
+/doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/command-ref/new-cli
@@ -26,6 +26,7 @@ perl/Makefile.config
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
+/doc/manual/src/language/builtin-constants.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix
new file mode 100644
index 000000000..3fc1fae42
--- /dev/null
+++ b/doc/manual/generate-builtin-constants.nix
@@ -0,0 +1,29 @@
+let
+  inherit (builtins) concatStringsSep attrValues mapAttrs;
+  inherit (import ./utils.nix) optionalString squash;
+in
+
+builtinsInfo:
+let
+  showBuiltin = name: { doc, type, impure-only }:
+    let
+      type' = optionalString (type != null) " (${type})";
+
+      impureNotice = optionalString impure-only ''
+        Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+      '';
+    in
+    squash ''
+      <dt id="builtin-constants-${name}">
+        <a href="#builtin-constants-${name}"><code>${name}</code>${type'}</a>
+      </dt>
+      <dd>
+
+      ${doc}
+
+      ${impureNotice}
+
+      </dd>
+    '';
+in
+concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))
diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 71f96153f..813a287f5 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -1,24 +1,28 @@
 let
-  inherit (builtins) concatStringsSep attrNames;
+  inherit (builtins) concatStringsSep attrValues mapAttrs;
+  inherit (import ./utils.nix) optionalString squash;
 in
 
 builtinsInfo:
 let
-  showBuiltin = name:
+  showBuiltin = name: { doc, args, arity, experimental-feature }:
     let
-      inherit (builtinsInfo.${name}) doc args;
+      experimentalNotice = optionalString (experimental-feature != null) ''
+        This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled.
+      '';
     in
-    ''
+    squash ''
       <dt id="builtins-${name}">
         <a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a>
       </dt>
       <dd>
 
-        ${doc}
+      ${doc}
+
+      ${experimentalNotice}
 
       </dd>
     '';
   listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args);
 in
-concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo))
-
+concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index b4b7283ef..abdfd6a62 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -128,14 +128,20 @@ $(d)/xp-features.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
+$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/builtins.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
+$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
+	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
+	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
+	@mv $@.tmp $@
+
+$(d)/language.json: $(bindir)/nix
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
 # Generate the HTML manual.
@@ -167,7 +173,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md
new file mode 100644
index 000000000..50f43006d
--- /dev/null
+++ b/doc/manual/src/language/builtin-constants-prefix.md
@@ -0,0 +1,5 @@
+# Built-in Constants
+
+These constants are built into the Nix language evaluator:
+
+<dl>
diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md
new file mode 100644
index 000000000..a74db2857
--- /dev/null
+++ b/doc/manual/src/language/builtin-constants-suffix.md
@@ -0,0 +1 @@
+</dl>
diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md
deleted file mode 100644
index 99520ad3f..000000000
--- a/doc/manual/src/language/builtin-constants.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# Built-in Constants
-
-These constants are built into the Nix language evaluator:
-
-- [`builtins`]{#builtins-builtins} (attribute set)
-
-  Contains all the [built-in functions](./builtins.md) and values.
-
-  Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
-
-  ```nix
-  # if hasContext is not available, we assume `s` has a context
-  if builtins ? hasContext then builtins.hasContext s else true
-  ```
-
-- [`builtins.currentSystem`]{#builtins-currentSystem} (string)
-
-  The built-in value `currentSystem` evaluates to the Nix platform
-  identifier for the Nix installation on which the expression is being
-  evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
-
-  Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
-
-- [`builtins.currentTime`]{#builtins-currentTime} (integer)
-
-  Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
-  Repeated references to that name will re-use the initially obtained value.
-
-  Example:
-
-  ```console
-  $ nix repl
-  Welcome to Nix 2.15.1 Type :? for help.
-
-  nix-repl> builtins.currentTime
-  1683705525
-
-  nix-repl> builtins.currentTime
-  1683705525
-  ```
-
-  The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation.
-
-  Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 8a0ff4cce..aaeb419ad 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -703,28 +703,34 @@ Path EvalState::toRealPath(const Path & path, const NixStringContext & context)
 }
 
 
-Value * EvalState::addConstant(const std::string & name, Value & v)
+Value * EvalState::addConstant(const std::string & name, Value & v, Constant info)
 {
     Value * v2 = allocValue();
     *v2 = v;
-    addConstant(name, v2);
+    addConstant(name, v2, info);
     return v2;
 }
 
 
-void EvalState::addConstant(const std::string & name, Value * v)
+void EvalState::addConstant(const std::string & name, Value * v, Constant info)
 {
-    staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
-    baseEnv.values[baseEnvDispl++] = v;
     auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name;
-    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
-}
 
+    constantInfos.push_back({name2, info});
 
-Value * EvalState::addPrimOp(const std::string & name,
-    size_t arity, PrimOpFun primOp)
-{
-    return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name });
+    if (!(evalSettings.pureEval && info.impureOnly)) {
+        /* Check the type, if possible.
+
+           We might know the type of a thunk in advance, so be allowed
+           to just write it down in that case. */
+        if (auto gotType = v->type(true); gotType != nThunk)
+            assert(info.type == gotType);
+
+        /* Install value the base environment. */
+        staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
+        baseEnv.values[baseEnvDispl++] = v;
+        baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
+    }
 }
 
 
@@ -738,7 +744,10 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         vPrimOp->mkPrimOp(new PrimOp(primOp));
         Value v;
         v.mkApp(vPrimOp, vPrimOp);
-        return addConstant(primOp.name, v);
+        return addConstant(primOp.name, v, {
+            .type = nThunk, // FIXME
+            .doc = primOp.doc,
+        });
     }
 
     auto envName = symbols.create(primOp.name);
@@ -764,13 +773,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        if (v2->primOp->doc)
+        if (auto * doc = v2->primOp->doc)
             return Doc {
                 .pos = {},
                 .name = v2->primOp->name,
                 .arity = v2->primOp->arity,
                 .args = v2->primOp->args,
-                .doc = v2->primOp->doc,
+                .doc = doc,
             };
     }
     return {};
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 0c07ae081..e3676c1b7 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -25,15 +25,72 @@ struct DerivedPath;
 enum RepairFlag : bool;
 
 
+/**
+ * Function that implements a primop.
+ */
 typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
+/**
+ * Info about a primitive operation, and its implementation
+ */
 struct PrimOp
 {
-    PrimOpFun fun;
-    size_t arity;
+    /**
+     * Name of the primop. `__` prefix is treated specially.
+     */
     std::string name;
+
+    /**
+     * Names of the parameters of a primop, for primops that take a
+     * fixed number of arguments to be substituted for these parameters.
+     */
     std::vector<std::string> args;
+
+    /**
+     * Aritiy of the primop.
+     *
+     * If `args` is not empty, this field will be computed from that
+     * field instead, so it doesn't need to be manually set.
+     */
+    size_t arity = 0;
+
+    /**
+     * Optional free-form documentation about the primop.
+     */
     const char * doc = nullptr;
+
+    /**
+     * Implementation of the primop.
+     */
+    PrimOpFun fun;
+
+    /**
+     * Optional experimental for this to be gated on.
+     */
+    std::optional<ExperimentalFeature> experimentalFeature;
+};
+
+/**
+ * Info about a constant
+ */
+struct Constant
+{
+    /**
+     * Optional type of the constant (known since it is a fixed value).
+     *
+     * @todo we should use an enum for this.
+     */
+    ValueType type = nThunk;
+
+    /**
+     * Optional free-form documentation about the constant.
+     */
+    const char * doc = nullptr;
+
+    /**
+     * Whether the constant is impure, and not available in pure mode.
+     */
+    bool impureOnly = false;
 };
 
 #if HAVE_BOEHMGC
@@ -513,18 +570,23 @@ public:
      */
     std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
 
+    /**
+     * Name and documentation about every constant.
+     *
+     * Constants from primops are hard to crawl, and their docs will go
+     * here too.
+     */
+    std::vector<std::pair<std::string, Constant>> constantInfos;
+
 private:
 
     unsigned int baseEnvDispl = 0;
 
     void createBaseEnv();
 
-    Value * addConstant(const std::string & name, Value & v);
+    Value * addConstant(const std::string & name, Value & v, Constant info);
 
-    void addConstant(const std::string & name, Value * v);
-
-    Value * addPrimOp(const std::string & name,
-        size_t arity, PrimOpFun primOp);
+    void addConstant(const std::string & name, Value * v, Constant info);
 
     Value * addPrimOp(PrimOp && primOp);
 
@@ -538,6 +600,10 @@ public:
         std::optional<std::string> name;
         size_t arity;
         std::vector<std::string> args;
+        /**
+         * Unlike the other `doc` fields in this file, this one should never be
+         * `null`.
+         */
         const char * doc;
     };
 
@@ -740,7 +806,12 @@ struct EvalSettings : Config
 
     Setting<Strings> nixPath{
         this, getDefaultNixPath(), "nix-path",
-        "List of directories to be searched for `<...>` file references."};
+        R"(
+          List of directories to be searched for `<...>` file references
+
+          In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
+          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath).
+        )"};
 
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 60bb6a71e..5aa44d6a1 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -788,9 +788,6 @@ static RegisterPrimOp r2({
       ```nix
       (builtins.getFlake "github:edolstra/dwarffs").rev
       ```
-
-      This function is only available if you enable the experimental feature
-      `flakes`.
     )",
     .fun = prim_getFlake,
     .experimentalFeature = Xp::Flakes,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9dd7bae02..5dfad470a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -238,7 +238,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
     }
 }
 
-static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info {
+static RegisterPrimOp primop_scopedImport(PrimOp {
     .name = "scopedImport",
     .arity = 2,
     .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -692,7 +692,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
         v.listElems()[n++] = i;
 }
 
-static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
+static RegisterPrimOp primop_genericClosure(PrimOp {
     .name = "__genericClosure",
     .args = {"attrset"},
     .arity = 1,
@@ -809,7 +809,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
     }
 }
 
-static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info {
+static RegisterPrimOp primop_addErrorContext(PrimOp {
     .name = "__addErrorContext",
     .arity = 2,
     .fun = prim_addErrorContext,
@@ -1400,7 +1400,7 @@ drvName, Bindings * attrs, Value & v)
     v.mkAttrs(result);
 }
 
-static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info {
+static RegisterPrimOp primop_derivationStrict(PrimOp {
     .name = "derivationStrict",
     .arity = 1,
     .fun = prim_derivationStrict,
@@ -1667,9 +1667,52 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
     v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos)));
 }
 
-static RegisterPrimOp primop_findFile(RegisterPrimOp::Info {
+static RegisterPrimOp primop_findFile(PrimOp {
     .name = "__findFile",
-    .arity = 2,
+    .args = {"search path", "lookup path"},
+    .doc = R"(
+      Look up the given path with the given search path.
+
+      A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`.
+      `prefix` is a relative path.
+      `path` denotes a file system location; the exact syntax depends on the command line interface.
+
+      Examples of search path attribute sets:
+
+      - ```
+        {
+          prefix = "nixos-config";
+          path = "/etc/nixos/configuration.nix";
+        }
+        ```
+
+      - ```
+        {
+          prefix = "";
+          path = "/nix/var/nix/profiles/per-user/root/channels";
+        }
+        ```
+
+      The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match.
+
+      This is the process for each entry:
+      If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`.
+      Note that the `path` may need to be downloaded at this point to look inside.
+      If the suffix is found inside that directory, then the entry is a match;
+      the combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
+
+      The syntax
+
+      ```nix
+      <nixpkgs>
+      ```
+
+      is equivalent to:
+
+      ```nix
+      builtins.findFile builtins.nixPath "nixpkgs"
+      ```
+    )",
     .fun = prim_findFile,
 });
 
@@ -2388,7 +2431,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * *
         state.mkPos(v, i->pos);
 }
 
-static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info {
+static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
     .name = "__unsafeGetAttrPos",
     .arity = 2,
     .fun = prim_unsafeGetAttrPos,
@@ -4061,10 +4104,10 @@ static RegisterPrimOp primop_splitVersion({
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(Info && info)
+RegisterPrimOp::RegisterPrimOp(PrimOp && primOp)
 {
     if (!primOps) primOps = new PrimOps;
-    primOps->push_back(std::move(info));
+    primOps->push_back(std::move(primOp));
 }
 
 
@@ -4077,54 +4120,202 @@ void EvalState::createBaseEnv()
 
     /* `builtins' must be first! */
     v.mkAttrs(buildBindings(128).finish());
-    addConstant("builtins", v);
+    addConstant("builtins", v, {
+        .type = nAttrs,
+        .doc = R"(
+          Contains all the [built-in functions](@docroot@/language/builtins.md) and values.
+
+          Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
+
+          ```nix
+          # if hasContext is not available, we assume `s` has a context
+          if builtins ? hasContext then builtins.hasContext s else true
+          ```
+        )",
+    });
 
     v.mkBool(true);
-    addConstant("true", v);
+    addConstant("true", v, {
+        .type = nBool,
+        .doc = R"(
+          Primitive value.
+
+          It can be returned by
+          [comparison operators](@docroot@/language/operators.md#Comparison)
+          and used in
+          [conditional expressions](@docroot@/language/constructs.md#Conditionals).
+
+          The name `true` is not special, and can be shadowed:
+
+          ```nix-repl
+          nix-repl> let true = 1; in true
+          1
+          ```
+        )",
+    });
 
     v.mkBool(false);
-    addConstant("false", v);
+    addConstant("false", v, {
+        .type = nBool,
+        .doc = R"(
+          Primitive value.
+
+          It can be returned by
+          [comparison operators](@docroot@/language/operators.md#Comparison)
+          and used in
+          [conditional expressions](@docroot@/language/constructs.md#Conditionals).
+
+          The name `false` is not special, and can be shadowed:
+
+          ```nix-repl
+          nix-repl> let false = 1; in false
+          1
+          ```
+        )",
+    });
 
     v.mkNull();
-    addConstant("null", v);
+    addConstant("null", v, {
+        .type = nNull,
+        .doc = R"(
+          Primitive value.
+
+          The name `null` is not special, and can be shadowed:
+
+          ```nix-repl
+          nix-repl> let null = 1; in null
+          1
+          ```
+        )",
+    });
 
     if (!evalSettings.pureEval) {
         v.mkInt(time(0));
-        addConstant("__currentTime", v);
-
-        v.mkString(settings.thisSystem.get());
-        addConstant("__currentSystem", v);
     }
+    addConstant("__currentTime", v, {
+        .type = nInt,
+        .doc = R"(
+          Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
+          Repeated references to that name will re-use the initially obtained value.
+
+          Example:
+
+          ```console
+          $ nix repl
+          Welcome to Nix 2.15.1 Type :? for help.
+
+          nix-repl> builtins.currentTime
+          1683705525
+
+          nix-repl> builtins.currentTime
+          1683705525
+          ```
+
+          The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
+        )",
+        .impureOnly = true,
+    });
+
+    if (!evalSettings.pureEval) {
+        v.mkString(settings.thisSystem.get());
+    }
+    addConstant("__currentSystem", v, {
+        .type = nString,
+        .doc = R"(
+          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+
+          It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
+
+          ```nix
+           builtins.derivation {
+             # ...
+             system = builtins.currentSystem;
+          }
+          ```
+
+          It can be overridden in order to create derivations for different system than the current one:
+
+          ```console
+          $ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem'
+          "mips64-linux"
+          ```
+        )",
+        .impureOnly = true,
+    });
 
     v.mkString(nixVersion);
-    addConstant("__nixVersion", v);
+    addConstant("__nixVersion", v, {
+        .type = nString,
+        .doc = R"(
+          The version of Nix.
+
+          For example, where the command line returns the current Nix version,
+
+          ```shell-session
+          $ nix --version
+          nix (Nix) 2.16.0
+          ```
+
+          the Nix language evaluator returns the same value:
+
+          ```nix-repl
+          nix-repl> builtins.nixVersion
+          "2.16.0"
+          ```
+        )",
+    });
 
     v.mkString(store->storeDir);
-    addConstant("__storeDir", v);
+    addConstant("__storeDir", v, {
+        .type = nString,
+        .doc = R"(
+          Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
+
+          This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
+
+          ```shell-session
+          $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir
+          "/blah"
+          ```
+        )",
+    });
 
     /* Language version.  This should be increased every time a new
        language feature gets added.  It's not necessary to increase it
        when primops get added, because you can just use `builtins ?
        primOp' to check. */
     v.mkInt(6);
-    addConstant("__langVersion", v);
+    addConstant("__langVersion", v, {
+        .type = nInt,
+        .doc = R"(
+          The current version of the Nix language.
+        )",
+    });
 
     // Miscellaneous
     if (evalSettings.enableNativeCode) {
-        addPrimOp("__importNative", 2, prim_importNative);
-        addPrimOp("__exec", 1, prim_exec);
+        addPrimOp({
+            .name = "__importNative",
+            .arity = 2,
+            .fun = prim_importNative,
+        });
+        addPrimOp({
+            .name = "__exec",
+            .arity = 1,
+            .fun = prim_exec,
+        });
     }
 
     addPrimOp({
-        .fun = evalSettings.traceVerbose ? prim_trace : prim_second,
-        .arity = 2,
         .name = "__traceVerbose",
         .args = { "e1", "e2" },
+        .arity = 2,
         .doc = R"(
           Evaluate *e1* and print its abstract syntax representation on standard
           error if `--trace-verbose` is enabled. Then return *e2*. This function
           is useful for debugging.
         )",
+        .fun = evalSettings.traceVerbose ? prim_trace : prim_second,
     });
 
     /* Add a value containing the current Nix expression search path. */
@@ -4136,26 +4327,46 @@ void EvalState::createBaseEnv()
         attrs.alloc("prefix").mkString(i.prefix);
         (v.listElems()[n++] = allocValue())->mkAttrs(attrs);
     }
-    addConstant("__nixPath", v);
+    addConstant("__nixPath", v, {
+        .type = nList,
+        .doc = R"(
+          The search path used to resolve angle bracket path lookups.
+
+          Angle bracket expressions can be
+          [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar)
+          using this and
+          [`builtins.findFile`](./builtins.html#builtins-findFile):
+
+          ```nix
+          <nixpkgs>
+          ```
+
+          is equivalent to:
+
+          ```nix
+          builtins.findFile builtins.nixPath "nixpkgs"
+          ```
+        )",
+    });
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            if (!primOp.experimentalFeature
-                || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature))
+            if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature))
             {
-                addPrimOp({
-                    .fun = primOp.fun,
-                    .arity = std::max(primOp.args.size(), primOp.arity),
-                    .name = primOp.name,
-                    .args = primOp.args,
-                    .doc = primOp.doc,
-                });
+                auto primOpAdjusted = primOp;
+                primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity);
+                addPrimOp(std::move(primOpAdjusted));
             }
 
     /* Add a wrapper around the derivation primop that computes the
-       `drvPath' and `outPath' attributes lazily. */
+       `drvPath' and `outPath' attributes lazily.
+
+       Null docs because it is documented separately.
+       */
     auto vDerivation = allocValue();
-    addConstant("derivation", vDerivation);
+    addConstant("derivation", vDerivation, {
+        .type = nFunction,
+    });
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 73b7b866c..930e7f32a 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -10,17 +10,7 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    struct Info
-    {
-        std::string name;
-        std::vector<std::string> args;
-        size_t arity = 0;
-        const char * doc;
-        PrimOpFun fun;
-        std::optional<ExperimentalFeature> experimentalFeature;
-    };
-
-    typedef std::vector<Info> PrimOps;
+    typedef std::vector<PrimOp> PrimOps;
     static PrimOps * primOps;
 
     /**
@@ -28,7 +18,7 @@ struct RegisterPrimOp
      * will get called during EvalState initialization, so there
      * may be primops not yet added and builtins is not yet sorted.
      */
-    RegisterPrimOp(Info && info);
+    RegisterPrimOp(PrimOp && primOp);
 };
 
 /* These primops are disabled without enableNativeCode, but plugins
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 4cf1f1e0b..bae849f61 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -154,9 +154,6 @@ static RegisterPrimOp primop_fetchClosure({
       specifying a binary cache from which the path can be fetched.
       Also, requiring a content-addressed final store path avoids the
       need for users to configure binary cache public keys.
-
-      This function is only available if you enable the experimental
-      feature `fetch-closure`.
     )",
     .fun = prim_fetchClosure,
     .experimentalFeature = Xp::FetchClosure,
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 89c0c36fd..44c8071c2 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -218,8 +218,11 @@ public:
     /**
      * Returns the normal type of a Value. This only returns nThunk if
      * the Value hasn't been forceValue'd
+     *
+     * @param invalidIsThunk Instead of aborting an an invalid (probably
+     * 0, so uninitialized) internal type, return `nThunk`.
      */
-    inline ValueType type() const
+    inline ValueType type(bool invalidIsThunk = false) const
     {
         switch (internalType) {
             case tInt: return nInt;
@@ -234,7 +237,10 @@ public:
             case tFloat: return nFloat;
             case tThunk: case tApp: case tBlackhole: return nThunk;
         }
-        abort();
+        if (invalidIsThunk)
+            return nThunk;
+        else
+            abort();
     }
 
     /**
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ce0bed2a3..650c79d14 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -352,7 +352,7 @@ void mainWrapped(int argc, char * * argv)
         return;
     }
 
-    if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+    if (argc == 2 && std::string(argv[1]) == "__dump-language") {
         experimentalFeatureSettings.experimentalFeatures = {
             Xp::Flakes,
             Xp::FetchClosure,
@@ -360,17 +360,34 @@ void mainWrapped(int argc, char * * argv)
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));
         auto res = nlohmann::json::object();
-        auto builtins = state.baseEnv.values[0]->attrs;
-        for (auto & builtin : *builtins) {
-            auto b = nlohmann::json::object();
-            if (!builtin.value->isPrimOp()) continue;
-            auto primOp = builtin.value->primOp;
-            if (!primOp->doc) continue;
-            b["arity"] = primOp->arity;
-            b["args"] = primOp->args;
-            b["doc"] = trim(stripIndentation(primOp->doc));
-            res[state.symbols[builtin.name]] = std::move(b);
-        }
+        res["builtins"] = ({
+            auto builtinsJson = nlohmann::json::object();
+            auto builtins = state.baseEnv.values[0]->attrs;
+            for (auto & builtin : *builtins) {
+                auto b = nlohmann::json::object();
+                if (!builtin.value->isPrimOp()) continue;
+                auto primOp = builtin.value->primOp;
+                if (!primOp->doc) continue;
+                b["arity"] = primOp->arity;
+                b["args"] = primOp->args;
+                b["doc"] = trim(stripIndentation(primOp->doc));
+                b["experimental-feature"] = primOp->experimentalFeature;
+                builtinsJson[state.symbols[builtin.name]] = std::move(b);
+            }
+            std::move(builtinsJson);
+        });
+        res["constants"] = ({
+            auto constantsJson = nlohmann::json::object();
+            for (auto & [name, info] : state.constantInfos) {
+                auto c = nlohmann::json::object();
+                if (!info.doc) continue;
+                c["doc"] = trim(stripIndentation(info.doc));
+                c["type"] = showType(info.type, false);
+                c["impure-only"] = info.impureOnly;
+                constantsJson[name] = std::move(c);
+            }
+            std::move(constantsJson);
+        });
         logger->cout("%s", res);
         return;
     }

From 80c9259756811c1165167db1bb66c1fef0accb65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= <jfroche@pyxel.be>
Date: Tue, 27 Jun 2023 12:01:08 +0200
Subject: [PATCH 149/183] Allow to sign path as unprivileged user

User can now sign path as unprivileged/allowed user

refs #1708
---
 doc/manual/src/release-notes/rl-next.md |  3 +++
 src/libstore/daemon.cc                  |  2 --
 tests/nixos/authorization.nix           | 15 +++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index bde9057c6..8479b166a 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,3 +1,6 @@
 # Release X.Y (202?-??-??)
 
 - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand
+
+- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
+  Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 75c3d2aca..ad3dee1a2 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -864,8 +864,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         StringSet sigs = readStrings<StringSet>(from);
         logger->startWork();
-        if (!trusted)
-            throw Error("you are not privileged to add signatures");
         store->addSignatures(path, sigs);
         logger->stopWork();
         to << 1;
diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix
index 7e8744dd9..fdeae06ed 100644
--- a/tests/nixos/authorization.nix
+++ b/tests/nixos/authorization.nix
@@ -75,5 +75,20 @@
       su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2
       grep -F "you are not privileged to repair paths" diag
     """)
+
+    machine.succeed("""
+        set -x
+        su --login mallory -c '
+          nix-store --generate-binary-cache-key cache1.example.org sk1 pk1
+          (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2
+        grep -F "cannot open connection to remote store 'daemon'" diag
+    """)
+
+    machine.succeed("""
+        su --login bob -c '
+          nix-store --generate-binary-cache-key cache1.example.org sk1 pk1
+          nix store sign --key-file sk1 ${pathFour}
+        '
+    """)
   '';
 }

From 2ccc02515f4a52d59f3473493f254942209fc81b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 27 Jun 2023 18:22:42 -0400
Subject: [PATCH 150/183] Trailing commas in redirects

This avoids diff noise when more are added. Unlike with JSON, this is
allowed in JS.
---
 doc/manual/redirects.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index d75cd231a..f54f6ea6a 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -330,23 +330,23 @@ const redirects = {
     "ssec-relnotes-2.0": "release-notes/rl-2.0.html",
     "ssec-relnotes-2.1": "release-notes/rl-2.1.html",
     "ssec-relnotes-2.2": "release-notes/rl-2.2.html",
-    "ssec-relnotes-2.3": "release-notes/rl-2.3.html"
+    "ssec-relnotes-2.3": "release-notes/rl-2.3.html",
   },
   "language/values.html": {
     "simple-values": "#primitives",
     "lists": "#list",
     "strings": "#string",
     "lists": "#list",
-    "attribute-sets": "#attribute-set"
+    "attribute-sets": "#attribute-set",
   },
   "installation/installing-binary.html": {
     "linux": "uninstall.html#linux",
     "macos": "uninstall.html#macos",
-    "uninstalling": "uninstall.html"
+    "uninstalling": "uninstall.html",
   }
   "contributing/hacking.html": {
-    "nix-with-flakes": "#building-nix-with-flakes"
-    "classic-nix": "#building-nix"
+    "nix-with-flakes": "#building-nix-with-flakes",
+    "classic-nix": "#building-nix",
   }
 };
 

From ca49e13414b4d1d9b2cdd72e36920e28c4ef117e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 21 Jun 2023 23:33:33 -0400
Subject: [PATCH 151/183] Split testing into its own page in the contribution
 guide

`hacking.md` has gotten really big!
---
 doc/manual/redirects.js                |   8 ++
 doc/manual/src/SUMMARY.md.in           |   1 +
 doc/manual/src/contributing/hacking.md | 165 ------------------------
 doc/manual/src/contributing/testing.md | 167 +++++++++++++++++++++++++
 4 files changed, 176 insertions(+), 165 deletions(-)
 create mode 100644 doc/manual/src/contributing/testing.md

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index f54f6ea6a..dcdb5d6e9 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -347,6 +347,14 @@ const redirects = {
   "contributing/hacking.html": {
     "nix-with-flakes": "#building-nix-with-flakes",
     "classic-nix": "#building-nix",
+    "running-tests": "testing.html#running-tests",
+    "unit-tests": "testing.html#unit-tests",
+    "functional-tests": "testing.html#functional-tests",
+    "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests",
+    "integration-tests": "testing.html#integration-tests",
+    "installer-tests": "testing.html#installer-tests",
+    "one-time-setup": "testing.html#one-time-setup",
+    "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing",
   }
 };
 
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index bbb603713..1bd8fa774 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -104,6 +104,7 @@
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)
+  - [Testing](contributing/testing.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 6c6f2eb52..7b2440971 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -190,171 +190,6 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
 
-## Running tests
-
-### Unit-tests
-
-The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
-under `src/{library_name}/tests` using the
-[googletest](https://google.github.io/googletest/) and
-[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
-
-You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
-
-### Functional tests
-
-The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
-Each test is a bash script.
-
-The whole test suite can be run with:
-
-```shell-session
-$ make install && make installcheck
-ran test tests/foo.sh... [PASS]
-ran test tests/bar.sh... [PASS]
-...
-```
-
-Individual tests can be run with `make`:
-
-```shell-session
-$ make tests/${testName}.sh.test
-ran test tests/${testName}.sh... [PASS]
-```
-
-or without `make`:
-
-```shell-session
-$ ./mk/run-test.sh tests/${testName}.sh
-ran test tests/${testName}.sh... [PASS]
-```
-
-To see the complete output, one can also run:
-
-```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
-+ foo
-output from foo
-+ bar
-output from bar
-...
-```
-
-The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
-
-#### Debugging failing functional tests
-
-When a functional test fails, it usually does so somewhere in the middle of the script.
-
-To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
-
-For example, if the script looks like:
-
-```bash
-foo
-nix blah blub
-bar
-```
-edit it like so:
-
-```diff
- foo
--nix blah blub
-+gdb --args nix blah blub
- bar
-```
-
-Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
-
-```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
-...
-+ gdb blash blub
-GNU gdb (GDB) 12.1
-...
-(gdb)
-```
-
-One can debug the Nix invocation in all the usual ways.
-For example, enter `run` to start the Nix invocation.
-
-### Integration tests
-
-The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
-These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
-Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
-
-You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
-
-### Installer tests
-
-After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
-
-Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
-
-- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
-  - `x86_64-linux`
-  - `armv6l-linux`
-  - `armv7l-linux`
-  - `x86_64-darwin`
-
-- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
-
-#### One-time setup
-
-1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
-2. At cachix.org:
-    - Create or log in to an account.
-    - Create a Cachix cache using the format `<github-username>-nix-install-tests`.
-    - Navigate to the new cache > Settings > Auth Tokens.
-    - Generate a new Cachix auth token and copy the generated value.
-3. At github.com:
-    - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
-    - Name the secret `CACHIX_AUTH_TOKEN`.
-    - Paste the copied value of the Cachix cache auth token.
-
-#### Using the CI-generated installer for manual testing
-
-After the CI run completes, you can check the output to extract the installer URL:
-1. Click into the detailed view of the CI run.
-2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
-3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
-4. Copy the value of `install_url`
-5. To generate an install command, plug this `install_url` and your GitHub username into this template:
-
-    ```console
-    curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
-    ```
-
-<!-- #### Manually generating test installers
-
-There's obviously a manual way to do this, and it's still the only way for
-platforms that lack GA runners.
-
-I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
-sketch what I recall in case it encourages someone to fill in detail, but: I
-didn't know what I was doing at the time and had to fumble/ask around a lot--
-so I don't want to uphold any of it as "right". It may have been dumb or
-the _hard_ way from the getgo. Fundamentals may have changed since.
-
-Here's the build command I used to do this on and for x86_64-darwin:
-nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
-
-I used the stable out-link to make it easier to script the next steps:
-link=$(readlink /tmp/foo)
-cp $link/*-darwin.tar.xz ~/somewheres
-
-I've lost the last steps and am just going from memory:
-
-From here, I think I had to extract and modify the `install` script to point
-it at this tarball (which I scped to my own site, but it might make more sense
-to just share them locally). I extracted this script once and then just
-search/replaced in it for each new build.
-
-The installer now supports a `--tarball-url-prefix` flag which _may_ have
-solved this need?
--->
-
 ### Checking links in the manual
 
 The build checks for broken internal links.
diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md
new file mode 100644
index 000000000..e5f80a928
--- /dev/null
+++ b/doc/manual/src/contributing/testing.md
@@ -0,0 +1,167 @@
+# Running tests
+
+## Unit-tests
+
+The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
+under `src/{library_name}/tests` using the
+[googletest](https://google.github.io/googletest/) and
+[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
+
+You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
+
+## Functional tests
+
+The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+Each test is a bash script.
+
+The whole test suite can be run with:
+
+```shell-session
+$ make install && make installcheck
+ran test tests/foo.sh... [PASS]
+ran test tests/bar.sh... [PASS]
+...
+```
+
+Individual tests can be run with `make`:
+
+```shell-session
+$ make tests/${testName}.sh.test
+ran test tests/${testName}.sh... [PASS]
+```
+
+or without `make`:
+
+```shell-session
+$ ./mk/run-test.sh tests/${testName}.sh
+ran test tests/${testName}.sh... [PASS]
+```
+
+To see the complete output, one can also run:
+
+```shell-session
+$ ./mk/debug-test.sh tests/${testName}.sh
++ foo
+output from foo
++ bar
+output from bar
+...
+```
+
+The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
+
+### Debugging failing functional tests
+
+When a functional test fails, it usually does so somewhere in the middle of the script.
+
+To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
+
+For example, if the script looks like:
+
+```bash
+foo
+nix blah blub
+bar
+```
+edit it like so:
+
+```diff
+ foo
+-nix blah blub
++gdb --args nix blah blub
+ bar
+```
+
+Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
+
+```shell-session
+$ ./mk/debug-test.sh tests/${testName}.sh
+...
++ gdb blash blub
+GNU gdb (GDB) 12.1
+...
+(gdb)
+```
+
+One can debug the Nix invocation in all the usual ways.
+For example, enter `run` to start the Nix invocation.
+
+## Integration tests
+
+The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
+These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
+Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
+
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
+
+## Installer tests
+
+After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
+
+Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
+
+- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
+  - `x86_64-linux`
+  - `armv6l-linux`
+  - `armv7l-linux`
+  - `x86_64-darwin`
+
+- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
+
+### One-time setup
+
+1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
+2. At cachix.org:
+    - Create or log in to an account.
+    - Create a Cachix cache using the format `<github-username>-nix-install-tests`.
+    - Navigate to the new cache > Settings > Auth Tokens.
+    - Generate a new Cachix auth token and copy the generated value.
+3. At github.com:
+    - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
+    - Name the secret `CACHIX_AUTH_TOKEN`.
+    - Paste the copied value of the Cachix cache auth token.
+
+## Working on documentation
+
+### Using the CI-generated installer for manual testing
+
+After the CI run completes, you can check the output to extract the installer URL:
+1. Click into the detailed view of the CI run.
+2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
+3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
+4. Copy the value of `install_url`
+5. To generate an install command, plug this `install_url` and your GitHub username into this template:
+
+    ```console
+    curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
+    ```
+
+<!-- #### Manually generating test installers
+
+There's obviously a manual way to do this, and it's still the only way for
+platforms that lack GA runners.
+
+I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
+sketch what I recall in case it encourages someone to fill in detail, but: I
+didn't know what I was doing at the time and had to fumble/ask around a lot--
+so I don't want to uphold any of it as "right". It may have been dumb or
+the _hard_ way from the getgo. Fundamentals may have changed since.
+
+Here's the build command I used to do this on and for x86_64-darwin:
+nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
+
+I used the stable out-link to make it easier to script the next steps:
+link=$(readlink /tmp/foo)
+cp $link/*-darwin.tar.xz ~/somewheres
+
+I've lost the last steps and am just going from memory:
+
+From here, I think I had to extract and modify the `install` script to point
+it at this tarball (which I scped to my own site, but it might make more sense
+to just share them locally). I extracted this script once and then just
+search/replaced in it for each new build.
+
+The installer now supports a `--tarball-url-prefix` flag which _may_ have
+solved this need?
+-->
+

From 3468cbaf479e3a2e9b3c7d6b00b2cb1976cce43e Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Wed, 28 Jun 2023 22:38:44 +0800
Subject: [PATCH 152/183] libexpr: fix leaking `es2` in stripIndentation
 (parser.y)

---
 src/libexpr/parser.y | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 5c9597f8a..ed6421ced 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -275,7 +275,12 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols,
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
+    if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
+        auto *const result = (*es2)[0].second;
+        delete es2;
+        return result;
+    }
+    return new ExprConcatStrings(pos, true, es2);
 }
 
 

From 685f1bb3864d52b80e74f920a150f8ffc7a479c3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Fri, 30 Jun 2023 15:10:07 +0200
Subject: [PATCH 153/183] labeler.yml: tests -> with-tests

---
 .github/labeler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/labeler.yml b/.github/labeler.yml
index fce0d3aeb..12120bdb3 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -16,7 +16,7 @@
 "new-cli":
   - src/nix/**/*
 
-"tests":
+"with-tests":
   # Unit tests
   - src/*/tests/**/*
   # Functional and integration tests

From eebfe989a5dc3aac622b9b5f2edef4461d8968c1 Mon Sep 17 00:00:00 2001
From: Yueh-Shun Li <shamrocklee@posteo.net>
Date: Fri, 30 Jun 2023 21:11:59 +0800
Subject: [PATCH 154/183] linkOrCopy: Fallback upon cross-device link error
 (EXDEV)

Fix building derivations in local chroot store on OpenAFS,
where hard linking accross directories causes cross-device link error
(EXDEV).
---
 src/libstore/build/local-derivation-goal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index ea7c52098..ee66ee500 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -395,8 +395,9 @@ static void linkOrCopy(const Path & from, const Path & to)
            bind-mount in this case?
 
            It can also fail with EPERM in BeegFS v7 and earlier versions
+           or fail with EXDEV in OpenAFS
            which don't allow hard-links to other directories */
-        if (errno != EMLINK && errno != EPERM)
+        if (errno != EMLINK && errno != EPERM && errno != EXDEV)
             throw SysError("linking '%s' to '%s'", to, from);
         copyPath(from, to);
     }

From 0f6d596df5866d3f08e743ac0f1a282daf6b2155 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 19 May 2023 09:49:18 +0200
Subject: [PATCH 155/183] fetchClosure: Factor out attribute hint

---
 src/libexpr/primops/fetchClosure.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index bae849f61..26c2b90bf 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -16,11 +16,13 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     for (auto & attr : *args[0]->attrs) {
         const auto & attrName = state.symbols[attr.name];
+        auto attrHint = [&]() -> std::string {
+            return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";
+        };
 
         if (attrName == "fromPath") {
             NixStringContext context;
-            fromPath = state.coerceToStorePath(attr.pos, *attr.value, context,
-                    "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure");
+            fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
         }
 
         else if (attrName == "toPath") {
@@ -28,14 +30,13 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             toCA = true;
             if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
                 NixStringContext context;
-                toPath = state.coerceToStorePath(attr.pos, *attr.value, context,
-                        "while evaluating the 'toPath' attribute passed to builtins.fetchClosure");
+                toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
             }
         }
 
         else if (attrName == "fromStore")
             fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos,
-                    "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure");
+                    attrHint());
 
         else
             throw Error({

From 7e5b6d2c4550a3007946271768e114531d7b89f0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 19 May 2023 10:03:25 +0200
Subject: [PATCH 156/183] fetchClosure: Refactor: rename toCA ->
 enableRewriting

---
 src/libexpr/primops/fetchClosure.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 26c2b90bf..8f8e033a5 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -11,7 +11,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     std::optional<std::string> fromStoreUrl;
     std::optional<StorePath> fromPath;
-    bool toCA = false;
+    bool enableRewriting = false;
     std::optional<StorePath> toPath;
 
     for (auto & attr : *args[0]->attrs) {
@@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
         else if (attrName == "toPath") {
             state.forceValue(*attr.value, attr.pos);
-            toCA = true;
+            enableRewriting = true;
             if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
                 NixStringContext context;
                 toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
@@ -75,7 +75,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     auto fromStore = openStore(parsedURL.to_string());
 
-    if (toCA) {
+    if (enableRewriting) {
         if (!toPath || !state.store->isValidPath(*toPath)) {
             auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
             auto i = remappings.find(*fromPath);

From ea30f152b738fe65f216f3173d3a19adaaef5323 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 19 May 2023 10:48:53 +0200
Subject: [PATCH 157/183] fetchClosure: Allow input addressed paths in pure
 mode

When explicitly requested by the caller, as suggested in the meeting
(https://github.com/NixOS/nix/pull/8090#issuecomment-1531139324)

> @edolstra: { toPath } vs { fromPath } is too implicit

I've opted for the `inputAddressed = true` requirement, because it
we did not agree on renaming the path attributes.

> @roberth: more explicit
> @edolstra: except for the direction; not immediately clear in which direction the rewriting happens

This is in fact the most explicit syntax and a bit redundant, which is
good, because that redundancy lets us deliver an error message that
reminds expression authors that CA provides a better experience to
their users.
---
 src/libexpr/primops/fetchClosure.cc | 54 +++++++++++++++++++++++------
 tests/fetchClosure.sh               | 16 +++++++--
 2 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 8f8e033a5..96c1df544 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -13,6 +13,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
     std::optional<StorePath> fromPath;
     bool enableRewriting = false;
     std::optional<StorePath> toPath;
+    bool inputAddressed = false;
 
     for (auto & attr : *args[0]->attrs) {
         const auto & attrName = state.symbols[attr.name];
@@ -38,6 +39,9 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos,
                     attrHint());
 
+        else if (attrName == "inputAddressed")
+            inputAddressed = state.forceBool(*attr.value, attr.pos, attrHint());
+
         else
             throw Error({
                 .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName),
@@ -51,6 +55,18 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             .errPos = state.positions[pos]
         });
 
+    if (inputAddressed) {
+        if (toPath && toPath != fromPath)
+            throw Error({
+                .msg = hintfmt("attribute '%s' is set to true, but 'toPath' does not match 'fromPath'. 'toPath' should be equal, or should be omitted. Instead 'toPath' was '%s' and 'fromPath' was '%s'",
+                    "inputAddressed",
+                    state.store->printStorePath(*toPath),
+                    state.store->printStorePath(*fromPath)),
+                .errPos = state.positions[pos]
+            });
+        assert(!enableRewriting);
+    }
+
     if (!fromStoreUrl)
         throw Error({
             .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
@@ -104,15 +120,28 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
         toPath = fromPath;
     }
 
-    /* In pure mode, require a CA path. */
-    if (evalSettings.pureEval) {
+    /* We want input addressing to be explicit, to inform readers and to give
+       expression authors an opportunity to improve their user experience. */
+    if (!inputAddressed) {
         auto info = state.store->queryPathInfo(*toPath);
-        if (!info->isContentAddressed(*state.store))
-            throw Error({
-                .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
-                    state.store->printStorePath(*toPath)),
-                .errPos = state.positions[pos]
-            });
+        if (!info->isContentAddressed(*state.store)) {
+            if (enableRewriting) {
+                throw Error({
+                    // Ideally we'd compute the path for them, but this error message is unlikely to occur in practice, so we keep it simple.
+                    .msg = hintfmt("Rewriting was requested, but 'toPath' is not content addressed. This is impossible. Please change 'toPath' to the correct path, or to a non-existing path, and try again",
+                        state.store->printStorePath(*toPath)),
+                    .errPos = state.positions[pos]
+                });
+            } else {
+                // We just checked toPath, but we report fromPath, because that's what the user certainly passed.
+                assert (toPath == fromPath);
+                throw Error({
+                    .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
+                        state.store->printStorePath(*fromPath)),
+                    .errPos = state.positions[pos]
+                });
+            }
+        }
     }
 
     state.mkStorePathString(*toPath, v);
@@ -138,7 +167,7 @@ static RegisterPrimOp primop_fetchClosure({
       `/nix/store/ldbh...`.
 
       If `fromPath` is already content-addressed, or if you are
-      allowing impure evaluation (`--impure`), then `toPath` may be
+      allowing input addressing (`inputAddressed = true;`), then `toPath` may be
       omitted.
 
       To find out the correct value for `toPath` given a `fromPath`,
@@ -153,8 +182,11 @@ static RegisterPrimOp primop_fetchClosure({
       allows you to use a previously built store path in a Nix
       expression. However, it is more reproducible because it requires
       specifying a binary cache from which the path can be fetched.
-      Also, requiring a content-addressed final store path avoids the
-      need for users to configure binary cache public keys.
+      Also, the default requirement of a content-addressed final store path
+      avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys).
+
+      This function is only available if you enable the experimental
+      feature `fetch-closure`.
     )",
     .fun = prim_fetchClosure,
     .experimentalFeature = Xp::FetchClosure,
diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh
index 21650eb06..4b8198e94 100644
--- a/tests/fetchClosure.sh
+++ b/tests/fetchClosure.sh
@@ -35,16 +35,28 @@ clearStore
 
 if [[ "$NIX_REMOTE" != "daemon" ]]; then
 
-    # In impure mode, we can use non-CA paths.
-    [[ $(nix eval --raw --no-require-sigs --impure --expr "
+    # We can use non-CA paths when we ask explicitly.
+    [[ $(nix eval --raw --no-require-sigs --expr "
       builtins.fetchClosure {
         fromStore = \"file://$cacheDir\";
         fromPath = $nonCaPath;
+        inputAddressed = true;
       }
     ") = $nonCaPath ]]
 
     [ -e $nonCaPath ]
 
+    # .. but only if we ask explicitly.
+    expectStderr 1 nix eval --raw --no-require-sigs --expr "
+      builtins.fetchClosure {
+        fromStore = \"file://$cacheDir\";
+        fromPath = $nonCaPath;
+      }
+    " | grepQuiet -E "The .fromPath. value .* is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add .inputAddressed = true;. to the .fetchClosure. arguments."
+
+    [ -e $nonCaPath ]
+
+
 fi
 
 # 'toPath' set to empty string should fail but print the expected path.

From 508aa58e671583bf06fa3597629839827c8d1bd7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 19 May 2023 16:25:23 +0200
Subject: [PATCH 158/183] fetchClosure: Always check that inputAddressed
 matches the result

---
 src/libexpr/primops/fetchClosure.cc | 42 +++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 96c1df544..c66fed633 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -13,7 +13,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
     std::optional<StorePath> fromPath;
     bool enableRewriting = false;
     std::optional<StorePath> toPath;
-    bool inputAddressed = false;
+    std::optional<bool> inputAddressedMaybe;
 
     for (auto & attr : *args[0]->attrs) {
         const auto & attrName = state.symbols[attr.name];
@@ -40,7 +40,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
                     attrHint());
 
         else if (attrName == "inputAddressed")
-            inputAddressed = state.forceBool(*attr.value, attr.pos, attrHint());
+            inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint());
 
         else
             throw Error({
@@ -55,6 +55,8 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             .errPos = state.positions[pos]
         });
 
+    bool inputAddressed = inputAddressedMaybe.value_or(false);
+
     if (inputAddressed) {
         if (toPath && toPath != fromPath)
             throw Error({
@@ -120,15 +122,39 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
         toPath = fromPath;
     }
 
-    /* We want input addressing to be explicit, to inform readers and to give
-       expression authors an opportunity to improve their user experience. */
-    if (!inputAddressed) {
-        auto info = state.store->queryPathInfo(*toPath);
+    auto info = state.store->queryPathInfo(*toPath);
+
+    /* If inputAddressed is set, make sure the result is as expected. */
+    if (inputAddressedMaybe) {
+        bool expectCA = !inputAddressed;
+        if (info->isContentAddressed(*state.store) != expectCA) {
+            if (inputAddressed)
+                throw Error({
+                    .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is ok or even preferable to return a content addressed path, so you probably want to remove the 'inputAddressed' attribute",
+                        state.store->printStorePath(*toPath)),
+                    .errPos = state.positions[pos]
+                });
+            else
+                throw Error({
+                    .msg = hintfmt("The 'fetchClosure' result, '%s' is input addressed, despite 'inputAddressed' being set to false. Please make sure you passed the correct path(s) or set 'inputAddressed' to true if you intended to return an input addressed path",
+                        state.store->printStorePath(*toPath)),
+                    .errPos = state.positions[pos]
+                });
+        }
+    } else {
+        /*
+          While it's fine to omit the inputAddressed attribute for CA paths,
+          we want input addressing to be explicit, to inform readers and to give
+          expression authors an opportunity to improve their user experience;
+          see message below.
+        */
         if (!info->isContentAddressed(*state.store)) {
             if (enableRewriting) {
+                // We don't perform the rewriting when outPath already exists, as an optimisation.
+                // However, we can quickly detect a mistake if the toPath is input addressed.
+                // Ideally we'd compute the path for them here, but this error message is unlikely to occur in practice, so we keep it simple.
                 throw Error({
-                    // Ideally we'd compute the path for them, but this error message is unlikely to occur in practice, so we keep it simple.
-                    .msg = hintfmt("Rewriting was requested, but 'toPath' is not content addressed. This is impossible. Please change 'toPath' to the correct path, or to a non-existing path, and try again",
+                    .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.",
                         state.store->printStorePath(*toPath)),
                     .errPos = state.positions[pos]
                 });

From 8dca95386c9ab8b13886716ec31e1b2936a3ef10 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 11:11:53 +0200
Subject: [PATCH 159/183] fetchClosure: Disallow toPath for inputAddressed =
 true

---
 src/libexpr/primops/fetchClosure.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index c66fed633..6d2271853 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -58,12 +58,11 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
     bool inputAddressed = inputAddressedMaybe.value_or(false);
 
     if (inputAddressed) {
-        if (toPath && toPath != fromPath)
+        if (toPath)
             throw Error({
-                .msg = hintfmt("attribute '%s' is set to true, but 'toPath' does not match 'fromPath'. 'toPath' should be equal, or should be omitted. Instead 'toPath' was '%s' and 'fromPath' was '%s'",
+                .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them",
                     "inputAddressed",
-                    state.store->printStorePath(*toPath),
-                    state.store->printStorePath(*fromPath)),
+                    "toPath"),
                 .errPos = state.positions[pos]
             });
         assert(!enableRewriting);

From 55888633dd323c83cd1db3773d82dc954da34888 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 11:37:41 +0200
Subject: [PATCH 160/183] makeContentAddressed: Add single path helper

---
 src/libexpr/primops/fetchClosure.cc    | 10 ++++------
 src/libstore/make-content-addressed.cc | 11 +++++++++++
 src/libstore/make-content-addressed.hh | 13 ++++++++++++-
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 6d2271853..6a15c826f 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -94,14 +94,12 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     if (enableRewriting) {
         if (!toPath || !state.store->isValidPath(*toPath)) {
-            auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
-            auto i = remappings.find(*fromPath);
-            assert(i != remappings.end());
-            if (toPath && *toPath != i->second)
+            auto rewrittenPath = makeContentAddressed(*fromStore, *state.store, *fromPath);
+            if (toPath && *toPath != rewrittenPath)
                 throw Error({
                     .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
                         state.store->printStorePath(*fromPath),
-                        state.store->printStorePath(i->second),
+                        state.store->printStorePath(rewrittenPath),
                         state.store->printStorePath(*toPath)),
                     .errPos = state.positions[pos]
                 });
@@ -111,7 +109,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
                         "rewriting '%s' to content-addressed form yielded '%s'; "
                         "please set this in the 'toPath' attribute passed to 'fetchClosure'",
                         state.store->printStorePath(*fromPath),
-                        state.store->printStorePath(i->second)),
+                        state.store->printStorePath(rewrittenPath)),
                     .errPos = state.positions[pos]
                 });
         }
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
index 53fe04704..626a22480 100644
--- a/src/libstore/make-content-addressed.cc
+++ b/src/libstore/make-content-addressed.cc
@@ -80,4 +80,15 @@ std::map<StorePath, StorePath> makeContentAddressed(
     return remappings;
 }
 
+StorePath makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePath & fromPath)
+{
+    auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath });
+    auto i = remappings.find(fromPath);
+    assert(i != remappings.end());
+    return i->second;
+}
+
 }
diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh
index 2ce6ec7bc..60bb2b477 100644
--- a/src/libstore/make-content-addressed.hh
+++ b/src/libstore/make-content-addressed.hh
@@ -5,9 +5,20 @@
 
 namespace nix {
 
+/** Rewrite a closure of store paths to be completely content addressed.
+ */
 std::map<StorePath, StorePath> makeContentAddressed(
     Store & srcStore,
     Store & dstStore,
-    const StorePathSet & storePaths);
+    const StorePathSet & rootPaths);
+
+/** Rewrite a closure of a store path to be completely content addressed.
+ *
+ * This is a convenience function for the case where you only have one root path.
+ */
+StorePath makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePath & rootPath);
 
 }

From 5bdca46117c701cd1cd47700c755367442537157 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 12:35:03 +0200
Subject: [PATCH 161/183] fetchClosure: Split into three cases

---
 src/libexpr/primops/fetchClosure.cc | 177 ++++++++++++++++------------
 1 file changed, 101 insertions(+), 76 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 6a15c826f..e8c32de96 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -5,6 +5,101 @@
 
 namespace nix {
 
+/**
+ * Handler for the content addressed case.
+ *
+ * @param state The evaluator state and store to write to.
+ * @param fromStore The store containing the path to rewrite.
+ * @param fromPath The source path to be rewritten.
+ * @param toPathMaybe The path to write the rewritten path to. If empty, the error shows the actual path.
+ * @param v The return `Value`
+ */
+static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional<StorePath> & toPathMaybe, Value &v) {
+
+    // establish toPath or throw
+
+    if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) {
+        auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath);
+        if (toPathMaybe && *toPathMaybe != rewrittenPath)
+            throw Error({
+                .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                    state.store->printStorePath(fromPath),
+                    state.store->printStorePath(rewrittenPath),
+                    state.store->printStorePath(*toPathMaybe)),
+                .errPos = state.positions[pos]
+            });
+        if (!toPathMaybe)
+            throw Error({
+                .msg = hintfmt(
+                    "rewriting '%s' to content-addressed form yielded '%s'; "
+                    "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                    state.store->printStorePath(fromPath),
+                    state.store->printStorePath(rewrittenPath)),
+                .errPos = state.positions[pos]
+            });
+    }
+
+    auto toPath = *toPathMaybe;
+
+    // check and return
+
+    auto resultInfo = state.store->queryPathInfo(toPath);
+
+    if (!resultInfo->isContentAddressed(*state.store)) {
+        // We don't perform the rewriting when outPath already exists, as an optimisation.
+        // However, we can quickly detect a mistake if the toPath is input addressed.
+        throw Error({
+            .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.",
+                state.store->printStorePath(toPath)),
+            .errPos = state.positions[pos]
+        });
+    }
+
+    state.mkStorePathString(toPath, v);
+}
+
+/**
+ * Fetch the closure and make sure it's content addressed.
+ */
+static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) {
+
+    if (!state.store->isValidPath(fromPath))
+        copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath });
+
+    auto info = state.store->queryPathInfo(fromPath);
+
+    if (!info->isContentAddressed(*state.store)) {
+        throw Error({
+            .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
+                state.store->printStorePath(fromPath)),
+            .errPos = state.positions[pos]
+        });
+    }
+
+    state.mkStorePathString(fromPath, v);
+}
+
+/**
+ * Fetch the closure and make sure it's input addressed.
+ */
+static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) {
+
+    if (!state.store->isValidPath(fromPath))
+        copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath });
+
+    auto info = state.store->queryPathInfo(fromPath);
+
+    if (info->isContentAddressed(*state.store)) {
+        throw Error({
+            .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is preferable to return a content addressed path, so remove the 'inputAddressed' attribute to ensure content addressing is used in the future",
+                state.store->printStorePath(fromPath)),
+            .errPos = state.positions[pos]
+        });
+    }
+
+    state.mkStorePathString(fromPath, v);
+}
+
 static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure");
@@ -92,82 +187,12 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     auto fromStore = openStore(parsedURL.to_string());
 
-    if (enableRewriting) {
-        if (!toPath || !state.store->isValidPath(*toPath)) {
-            auto rewrittenPath = makeContentAddressed(*fromStore, *state.store, *fromPath);
-            if (toPath && *toPath != rewrittenPath)
-                throw Error({
-                    .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
-                        state.store->printStorePath(*fromPath),
-                        state.store->printStorePath(rewrittenPath),
-                        state.store->printStorePath(*toPath)),
-                    .errPos = state.positions[pos]
-                });
-            if (!toPath)
-                throw Error({
-                    .msg = hintfmt(
-                        "rewriting '%s' to content-addressed form yielded '%s'; "
-                        "please set this in the 'toPath' attribute passed to 'fetchClosure'",
-                        state.store->printStorePath(*fromPath),
-                        state.store->printStorePath(rewrittenPath)),
-                    .errPos = state.positions[pos]
-                });
-        }
-    } else {
-        if (!state.store->isValidPath(*fromPath))
-            copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
-        toPath = fromPath;
-    }
-
-    auto info = state.store->queryPathInfo(*toPath);
-
-    /* If inputAddressed is set, make sure the result is as expected. */
-    if (inputAddressedMaybe) {
-        bool expectCA = !inputAddressed;
-        if (info->isContentAddressed(*state.store) != expectCA) {
-            if (inputAddressed)
-                throw Error({
-                    .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is ok or even preferable to return a content addressed path, so you probably want to remove the 'inputAddressed' attribute",
-                        state.store->printStorePath(*toPath)),
-                    .errPos = state.positions[pos]
-                });
-            else
-                throw Error({
-                    .msg = hintfmt("The 'fetchClosure' result, '%s' is input addressed, despite 'inputAddressed' being set to false. Please make sure you passed the correct path(s) or set 'inputAddressed' to true if you intended to return an input addressed path",
-                        state.store->printStorePath(*toPath)),
-                    .errPos = state.positions[pos]
-                });
-        }
-    } else {
-        /*
-          While it's fine to omit the inputAddressed attribute for CA paths,
-          we want input addressing to be explicit, to inform readers and to give
-          expression authors an opportunity to improve their user experience;
-          see message below.
-        */
-        if (!info->isContentAddressed(*state.store)) {
-            if (enableRewriting) {
-                // We don't perform the rewriting when outPath already exists, as an optimisation.
-                // However, we can quickly detect a mistake if the toPath is input addressed.
-                // Ideally we'd compute the path for them here, but this error message is unlikely to occur in practice, so we keep it simple.
-                throw Error({
-                    .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.",
-                        state.store->printStorePath(*toPath)),
-                    .errPos = state.positions[pos]
-                });
-            } else {
-                // We just checked toPath, but we report fromPath, because that's what the user certainly passed.
-                assert (toPath == fromPath);
-                throw Error({
-                    .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
-                        state.store->printStorePath(*fromPath)),
-                    .errPos = state.positions[pos]
-                });
-            }
-        }
-    }
-
-    state.mkStorePathString(*toPath, v);
+    if (enableRewriting)
+        runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, toPath, v);
+    else if (inputAddressed)
+        runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v);
+    else
+        runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v);
 }
 
 static RegisterPrimOp primop_fetchClosure({

From dc79636007ff6982300980ba7eb1c2be3e45bece Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 12:55:58 +0200
Subject: [PATCH 162/183] fetchClosure: Refactor: replace enableRewriting

A single variable is nice and self-contained.
---
 src/libexpr/primops/fetchClosure.cc | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index e8c32de96..e048a2629 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -100,14 +100,15 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId
     state.mkStorePathString(fromPath, v);
 }
 
+typedef std::optional<StorePath> StorePathOrGap;
+
 static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure");
 
     std::optional<std::string> fromStoreUrl;
     std::optional<StorePath> fromPath;
-    bool enableRewriting = false;
-    std::optional<StorePath> toPath;
+    std::optional<StorePathOrGap> toPath;
     std::optional<bool> inputAddressedMaybe;
 
     for (auto & attr : *args[0]->attrs) {
@@ -123,8 +124,11 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
         else if (attrName == "toPath") {
             state.forceValue(*attr.value, attr.pos);
-            enableRewriting = true;
-            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+            bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string("");
+            if (isEmptyString) {
+                toPath = StorePathOrGap {};
+            }
+            else {
                 NixStringContext context;
                 toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
             }
@@ -160,7 +164,6 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
                     "toPath"),
                 .errPos = state.positions[pos]
             });
-        assert(!enableRewriting);
     }
 
     if (!fromStoreUrl)
@@ -187,8 +190,8 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
     auto fromStore = openStore(parsedURL.to_string());
 
-    if (enableRewriting)
-        runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, toPath, v);
+    if (toPath)
+        runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);
     else if (inputAddressed)
         runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v);
     else

From 32c69e2b17ad7bd9721b584c3733629fe6d8c7f7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 16:25:02 +0200
Subject: [PATCH 163/183] doc: Typo

---
 doc/manual/src/language/builtins-prefix.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md
index 35e3dccc3..7b2321466 100644
--- a/doc/manual/src/language/builtins-prefix.md
+++ b/doc/manual/src/language/builtins-prefix.md
@@ -3,7 +3,7 @@
 This section lists the functions built into the Nix language evaluator.
 All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant.
 
-For convenience, some built-ins are can be accessed directly:
+For convenience, some built-ins can be accessed directly:
 
 - [`derivation`](#builtins-derivation)
 - [`import`](#builtins-import)

From 50de11d662ced6bfef792af54ad57553f1c3208a Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 5 Jun 2023 16:25:22 +0200
Subject: [PATCH 164/183] doc: Improve `fetchClosure` documentation

---
 doc/manual/src/release-notes/rl-next.md |  2 +
 src/libexpr/primops.cc                  |  2 +
 src/libexpr/primops/fetchClosure.cc     | 66 +++++++++++++++++++------
 3 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 8479b166a..e64839fa2 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -2,5 +2,7 @@
 
 - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand
 
+- The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure mode](../command-ref/conf-file.md#conf-pure-eval).
+
 - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
   Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5dfad470a..f4bdf8fc6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1502,6 +1502,8 @@ static RegisterPrimOp primop_storePath({
       in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
 
       Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+
+      See also [`builtins.fetchClosure`](#builtins-fetchClosure).
     )",
     .fun = prim_storePath,
 });
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index e048a2629..e344c9513 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -202,40 +202,76 @@ static RegisterPrimOp primop_fetchClosure({
     .name = "__fetchClosure",
     .args = {"args"},
     .doc = R"(
-      Fetch a Nix store closure from a binary cache, rewriting it into
-      content-addressed form. For example,
+      Fetch a Nix store closure from a binary cache.
 
-      ```nix
-      builtins.fetchClosure {
-        fromStore = "https://cache.nixos.org";
-        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
-        toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
-      }
-      ```
+      This function can be used in three ways:
 
-      fetches `/nix/store/r2jd...` from the specified binary cache,
+      - Fetch any store path and rewrite it to a fully content-addressed store path. Example:
+
+        ```nix
+        builtins.fetchClosure {
+            fromStore = "https://cache.nixos.org";
+            fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+            toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+        }
+        ```
+
+      - Fetch a content-addressed store path.
+
+        ```nix
+        builtins.fetchClosure {
+            fromStore = "https://cache.nixos.org";
+            fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+        }
+        ```
+
+      - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable.
+
+        ```nix
+        builtins.fetchClosure {
+            fromStore = "https://cache.nixos.org";
+            fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+            inputAddressed = true;
+        }
+        ```
+
+      **Rewriting a store path**
+
+      This first example fetches `/nix/store/r2jd...` from the specified binary cache,
       and rewrites it into the content-addressed store path
       `/nix/store/ldbh...`.
 
-      If `fromPath` is already content-addressed, or if you are
-      allowing input addressing (`inputAddressed = true;`), then `toPath` may be
-      omitted.
+      By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust.
 
       To find out the correct value for `toPath` given a `fromPath`,
-      you can use `nix store make-content-addressed`:
+      you can use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
 
       ```console
       # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
       rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
       ```
 
-      This function is similar to `builtins.storePath` in that it
+      Alternatively, you may set `toPath = ""` and find the correct `toPath` in the error message.
+
+      **Not rewriting**
+
+      `toPath` may be omitted when either
+       - `fromPath` is already content-addressed, or
+       - `inputAddressed = true;` is set.
+
+      Fetching an [input addressed path](@docroot@/glossary.md#gloss-input-addressed-store-object) is not recommended because it requires that the source be trusted by the Nix configuration.
+
+      **`builtins.storePath`**
+
+      This function is similar to [`builtins.storePath`](#builtins-storePath) in that it
       allows you to use a previously built store path in a Nix
       expression. However, it is more reproducible because it requires
       specifying a binary cache from which the path can be fetched.
       Also, the default requirement of a content-addressed final store path
       avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys).
 
+      **Experimental feature**
+
       This function is only available if you enable the experimental
       feature `fetch-closure`.
     )",

From 40052c76132d79561aa50ec9349ad22b51c64505 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Wed, 7 Jun 2023 12:47:18 +0200
Subject: [PATCH 165/183] fetchClosure: Docs and error message improvements

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/release-notes/rl-next.md |  2 +-
 src/libexpr/primops/fetchClosure.cc     | 38 ++++++++++++++++---------
 tests/fetchClosure.sh                   |  2 +-
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index e64839fa2..139d07188 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -2,7 +2,7 @@
 
 - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand
 
-- The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure mode](../command-ref/conf-file.md#conf-pure-eval).
+* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure.
 
 - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
   Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index e344c9513..5114df553 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -8,11 +8,11 @@ namespace nix {
 /**
  * Handler for the content addressed case.
  *
- * @param state The evaluator state and store to write to.
- * @param fromStore The store containing the path to rewrite.
- * @param fromPath The source path to be rewritten.
- * @param toPathMaybe The path to write the rewritten path to. If empty, the error shows the actual path.
- * @param v The return `Value`
+ * @param state Evaluator state and store to write to.
+ * @param fromStore Store containing the path to rewrite.
+ * @param fromPath Source path to be rewritten.
+ * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path.
+ * @param v Return `Value`
  */
 static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional<StorePath> & toPathMaybe, Value &v) {
 
@@ -31,8 +31,8 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
         if (!toPathMaybe)
             throw Error({
                 .msg = hintfmt(
-                    "rewriting '%s' to content-addressed form yielded '%s'; "
-                    "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                    "rewriting '%s' to content-addressed form yielded '%s'\n"
+                    "Use this value for the 'toPath' attribute passed to 'fetchClosure'",
                     state.store->printStorePath(fromPath),
                     state.store->printStorePath(rewrittenPath)),
                 .errPos = state.positions[pos]
@@ -49,7 +49,9 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
         // We don't perform the rewriting when outPath already exists, as an optimisation.
         // However, we can quickly detect a mistake if the toPath is input addressed.
         throw Error({
-            .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.",
+            .msg = hintfmt(
+                "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n"
+                "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.",
                 state.store->printStorePath(toPath)),
             .errPos = state.positions[pos]
         });
@@ -70,7 +72,9 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos
 
     if (!info->isContentAddressed(*state.store)) {
         throw Error({
-            .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
+            .msg = hintfmt(
+                "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n"
+                "If you do intend to fetch an input-addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
                 state.store->printStorePath(fromPath)),
             .errPos = state.positions[pos]
         });
@@ -91,7 +95,9 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId
 
     if (info->isContentAddressed(*state.store)) {
         throw Error({
-            .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is preferable to return a content addressed path, so remove the 'inputAddressed' attribute to ensure content addressing is used in the future",
+            .msg = hintfmt(
+                "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n"
+                "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed",
                 state.store->printStorePath(fromPath)),
             .errPos = state.positions[pos]
         });
@@ -202,11 +208,13 @@ static RegisterPrimOp primop_fetchClosure({
     .name = "__fetchClosure",
     .args = {"args"},
     .doc = R"(
-      Fetch a Nix store closure from a binary cache.
+      Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache.
 
       This function can be used in three ways:
 
-      - Fetch any store path and rewrite it to a fully content-addressed store path. Example:
+      - Fetch any store path and rewrite it to a fully content-addressed store path.
+
+        Example:
 
         ```nix
         builtins.fetchClosure {
@@ -218,6 +226,8 @@ static RegisterPrimOp primop_fetchClosure({
 
       - Fetch a content-addressed store path.
 
+Example:
+
         ```nix
         builtins.fetchClosure {
             fromStore = "https://cache.nixos.org";
@@ -227,6 +237,8 @@ static RegisterPrimOp primop_fetchClosure({
 
       - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable.
 
+        Example:
+
         ```nix
         builtins.fetchClosure {
             fromStore = "https://cache.nixos.org";
@@ -244,7 +256,7 @@ static RegisterPrimOp primop_fetchClosure({
       By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust.
 
       To find out the correct value for `toPath` given a `fromPath`,
-      you can use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
+      use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
 
       ```console
       # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh
index 4b8198e94..0f1c64372 100644
--- a/tests/fetchClosure.sh
+++ b/tests/fetchClosure.sh
@@ -52,7 +52,7 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then
         fromStore = \"file://$cacheDir\";
         fromPath = $nonCaPath;
       }
-    " | grepQuiet -E "The .fromPath. value .* is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add .inputAddressed = true;. to the .fetchClosure. arguments."
+    " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false."
 
     [ -e $nonCaPath ]
 

From 1db81f710725eb83597b1761076e9ca28317a52e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 19 Jun 2023 23:22:37 +0200
Subject: [PATCH 166/183] tests/fetchClosure: Improve coverage of new and some
 existing flows

---
 tests/fetchClosure.sh | 79 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 69 insertions(+), 10 deletions(-)

diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh
index 0f1c64372..a02d1ce7a 100644
--- a/tests/fetchClosure.sh
+++ b/tests/fetchClosure.sh
@@ -33,8 +33,26 @@ clearStore
 [ ! -e $nonCaPath ]
 [ -e $caPath ]
 
+clearStore
+
+# The daemon will reject input addressed paths unless configured to trust the
+# cache key or the user. This behavior should be covered by another test, so we
+# skip this part when using the daemon.
 if [[ "$NIX_REMOTE" != "daemon" ]]; then
 
+    # If we want to return a non-CA path, we have to be explicit about it.
+    expectStderr 1 nix eval --raw --no-require-sigs --expr "
+      builtins.fetchClosure {
+        fromStore = \"file://$cacheDir\";
+        fromPath = $nonCaPath;
+      }
+    " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false."
+
+    # TODO: Should the closure be rejected, despite single user mode?
+    # [ ! -e $nonCaPath ]
+
+    [ ! -e $caPath ]
+
     # We can use non-CA paths when we ask explicitly.
     [[ $(nix eval --raw --no-require-sigs --expr "
       builtins.fetchClosure {
@@ -45,20 +63,13 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then
     ") = $nonCaPath ]]
 
     [ -e $nonCaPath ]
-
-    # .. but only if we ask explicitly.
-    expectStderr 1 nix eval --raw --no-require-sigs --expr "
-      builtins.fetchClosure {
-        fromStore = \"file://$cacheDir\";
-        fromPath = $nonCaPath;
-      }
-    " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false."
-
-    [ -e $nonCaPath ]
+    [ ! -e $caPath ]
 
 
 fi
 
+[ ! -e $caPath ]
+
 # 'toPath' set to empty string should fail but print the expected path.
 expectStderr 1 nix eval -v --json --expr "
   builtins.fetchClosure {
@@ -71,6 +82,10 @@ expectStderr 1 nix eval -v --json --expr "
 # If fromPath is CA, then toPath isn't needed.
 nix copy --to file://$cacheDir $caPath
 
+clearStore
+
+[ ! -e $caPath ]
+
 [[ $(nix eval -v --raw --expr "
   builtins.fetchClosure {
     fromStore = \"file://$cacheDir\";
@@ -78,6 +93,8 @@ nix copy --to file://$cacheDir $caPath
   }
 ") = $caPath ]]
 
+[ -e $caPath ]
+
 # Check that URL query parameters aren't allowed.
 clearStore
 narCache=$TEST_ROOT/nar-cache
@@ -89,3 +106,45 @@ rm -rf $narCache
   }
 ")
 (! [ -e $narCache ])
+
+# If toPath is specified but wrong, we check it (only) when the path is missing.
+clearStore
+
+badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!')
+
+[ ! -e $badPath ]
+
+expectStderr 1 nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+    toPath = $badPath;
+  }
+" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected"
+
+[ ! -e $badPath ]
+
+# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it.
+# It would be nice for this to fail, but checking it would be too(?) slow.
+[ -e $caPath ]
+
+[[ $(nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $badPath;
+    toPath = $caPath;
+  }
+") = $caPath ]]
+
+
+# However, if the output address is unexpected, we can report it
+
+
+expectStderr 1 nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $caPath;
+    inputAddressed = true;
+  }
+" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*'
+

From fefb94713295d63d30a8bba257c41c3eb54e0998 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 19 Jun 2023 23:22:56 +0200
Subject: [PATCH 167/183] tests/signing.sh: Check signature checking error
 message

We should check error messages, so that we know the command fails for
the right reason.
Alternatively, a mere typo can run the test undetected.
---
 tests/signing.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/signing.sh b/tests/signing.sh
index 9b673c609..f14b53e7f 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -84,7 +84,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2)
 # Copying to a diverted store should fail due to a lack of signatures by trusted keys.
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0
-(! nix copy --to $TEST_ROOT/store0 $outPath)
+expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key'
 
 # But succeed if we supply the public keys.
 nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1

From a6c17097d2ee76a347d86c42a37fd6de0d0fd3ef Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 30 Jun 2023 23:36:27 +0200
Subject: [PATCH 168/183] tests: Don't install test-libstoreconsumer program

Sorry about that.
Fixes https://github.com/NixOS/nix/issues/8616
---
 tests/test-libstoreconsumer/local.mk | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk
index cd2d0c7f8..edc140723 100644
--- a/tests/test-libstoreconsumer/local.mk
+++ b/tests/test-libstoreconsumer/local.mk
@@ -2,6 +2,9 @@ programs += test-libstoreconsumer
 
 test-libstoreconsumer_DIR := $(d)
 
+# do not install
+test-libstoreconsumer_INSTALL_DIR :=
+
 test-libstoreconsumer_SOURCES := \
   $(wildcard $(d)/*.cc) \
 

From 87b82db8812736922de851a27681dd0b2955c2f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 May 2022 10:33:58 +0200
Subject: [PATCH 169/183] nix profile list: Improve readability of the output

---
 src/nix/profile-list.md | 46 ++++++++++++++++++++++++++++++-----------
 src/nix/profile.cc      | 14 +++++++++----
 2 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md
index fa786162f..5d7fcc0ec 100644
--- a/src/nix/profile-list.md
+++ b/src/nix/profile-list.md
@@ -6,26 +6,48 @@ R""(
 
   ```console
   # nix profile list
-  0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1
-  1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027
-  2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0
+  Index:              0
+  Flake attribute:    legacyPackages.x86_64-linux.gdb
+  Original flake URL: flake:nixpkgs
+  Locked flake URL:   github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca
+  Store paths:        /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1
+
+  Index:              1
+  Flake attribute:    packages.x86_64-linux.default
+  Original flake URL: flake:blender-bin
+  Locked flake URL:   github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender
+  Store paths:        /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2
   ```
 
+  Note that you can unambiguously rebuild a package from a profile
+  through its locked flake URL and flake attribute, e.g.
+
+  ```console
+  # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default
+  ```
+
+  will build the package with index 1 shown above.
+
 # Description
 
 This command shows what packages are currently installed in a
-profile. The output consists of one line per package, with the
-following fields:
+profile. For each installed package, it shows the following
+information:
 
-* An integer that can be used to unambiguously identify the package in
-  invocations of `nix profile remove` and `nix profile upgrade`.
+* `Index`: An integer that can be used to unambiguously identify the
+  package in invocations of `nix profile remove` and `nix profile
+  upgrade`.
 
-* The original ("unlocked") flake reference and output attribute path
-  used at installation time.
+* `Flake attribute`: The flake output attribute path that provides the
+  package (e.g. `packages.x86_64-linux.hello`).
 
-* The locked flake reference to which the unlocked flake reference was
-  resolved.
+* `Original flake URL`: The original ("unlocked") flake reference
+  specified by the user when the package was first installed via `nix
+  profile install`.
 
-* The store path(s) of the package.
+* `Locked flake URL`: The locked flake reference to which the original
+  flake reference was resolved.
+
+* `Store paths`: The store path(s) of the package.
 
 )""
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index f3b73f10d..b0cedb24a 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -655,10 +655,16 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
-            logger->cout("%d %s %s %s", i,
-                element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
-                element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
-                concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
+            if (i) logger->cout("");
+            logger->cout("Index:              " ANSI_BOLD "%s" ANSI_NORMAL "%s",
+                i,
+                element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
+            if (element.source) {
+                logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
+                logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
+                logger->cout("Locked flake URL:   %s", element.source->resolvedRef.to_string());
+            }
+            logger->cout("Store paths:        %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
         }
     }
 };

From 3c90340fe64fb18e66f02a648ebce8ad629482a5 Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Mon, 3 Jul 2023 16:02:54 +0800
Subject: [PATCH 170/183] libexpr: use `thread_local` to make the parser
 thread-safe

If we call `adjustLoc`, the global variable `prev_yylloc` is shared
between threads and racy.

Currently, nix itself does not concurrently parsing files, but this is
helpful for libexpr users. (The parser is thread-safe except this.)
---
 src/libexpr/lexer.l | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 462b3b602..a3a8608d9 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -36,7 +36,7 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
 #define CUR_POS makeCurPos(*yylloc, data)
 
 // backup to recover from yyless(0)
-YYLTYPE prev_yylloc;
+thread_local YYLTYPE prev_yylloc;
 
 static void initLoc(YYLTYPE * loc)
 {

From b8e8f27159251091141ccd6db12dd2d07204771a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 May 2022 10:35:28 +0200
Subject: [PATCH 171/183] Rename 'resolvedRef' to 'lockedRef'

'resolvedRef' was incorrect, since a resolved ref is one after
registry resolution, which may still be unlocked (e.g. 'nixpkgs' ->
'github:NixOS/nixpkgs').
---
 src/libcmd/installable-flake.cc |  2 +-
 src/libcmd/installable-flake.hh |  2 +-
 src/nix/profile.cc              | 14 +++++++-------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc
index eb944240b..4da9b131b 100644
--- a/src/libcmd/installable-flake.cc
+++ b/src/libcmd/installable-flake.cc
@@ -151,7 +151,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             },
             ExtraPathInfoFlake::Flake {
                 .originalRef = flakeRef,
-                .resolvedRef = getLockedFlake()->flake.lockedRef,
+                .lockedRef = getLockedFlake()->flake.lockedRef,
             }),
     }};
 }
diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh
index 7ac4358d2..314918c14 100644
--- a/src/libcmd/installable-flake.hh
+++ b/src/libcmd/installable-flake.hh
@@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue
      */
     struct Flake {
         FlakeRef originalRef;
-        FlakeRef resolvedRef;
+        FlakeRef lockedRef;
     };
 
     Flake flake;
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index b0cedb24a..ae2e8f4f1 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -21,7 +21,7 @@ struct ProfileElementSource
 {
     FlakeRef originalRef;
     // FIXME: record original attrpath.
-    FlakeRef resolvedRef;
+    FlakeRef lockedRef;
     std::string attrPath;
     ExtendedOutputsSpec outputs;
 
@@ -181,7 +181,7 @@ struct ProfileManifest
             obj["priority"] = element.priority;
             if (element.source) {
                 obj["originalUrl"] = element.source->originalRef.to_string();
-                obj["url"] = element.source->resolvedRef.to_string();
+                obj["url"] = element.source->lockedRef.to_string();
                 obj["attrPath"] = element.source->attrPath;
                 obj["outputs"] = element.source->outputs;
             }
@@ -349,7 +349,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             if (auto * info2 = dynamic_cast<ExtraPathInfoFlake *>(&*info)) {
                 element.source = ProfileElementSource {
                     .originalRef = info2->flake.originalRef,
-                    .resolvedRef = info2->flake.resolvedRef,
+                    .lockedRef = info2->flake.lockedRef,
                     .attrPath = info2->value.attrPath,
                     .outputs = info2->value.extendedOutputsSpec,
                 };
@@ -588,14 +588,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 assert(infop);
                 auto & info = *infop;
 
-                if (element.source->resolvedRef == info.flake.resolvedRef) continue;
+                if (element.source->lockedRef == info.flake.lockedRef) continue;
 
                 printInfo("upgrading '%s' from flake '%s' to '%s'",
-                    element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef);
+                    element.source->attrPath, element.source->lockedRef, info.flake.lockedRef);
 
                 element.source = ProfileElementSource {
                     .originalRef = installable->flakeRef,
-                    .resolvedRef = info.flake.resolvedRef,
+                    .lockedRef = info.flake.lockedRef,
                     .attrPath = info.value.attrPath,
                     .outputs = installable->extendedOutputsSpec,
                 };
@@ -662,7 +662,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
             if (element.source) {
                 logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
                 logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
-                logger->cout("Locked flake URL:   %s", element.source->resolvedRef.to_string());
+                logger->cout("Locked flake URL:   %s", element.source->lockedRef.to_string());
             }
             logger->cout("Store paths:        %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
         }

From a353412c43e776f4fdc709dc7affd1dbb9d588be Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 May 2022 10:44:26 +0200
Subject: [PATCH 172/183] nix profile list: Add --json flag

This just dumps the profile manifest to stdout.
---
 src/nix/profile.cc | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index ae2e8f4f1..b833b5192 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -168,7 +168,7 @@ struct ProfileManifest
         }
     }
 
-    std::string toJSON(Store & store) const
+    nlohmann::json toJSON(Store & store) const
     {
         auto array = nlohmann::json::array();
         for (auto & element : elements) {
@@ -190,7 +190,7 @@ struct ProfileManifest
         nlohmann::json json;
         json["version"] = 2;
         json["elements"] = array;
-        return json.dump();
+        return json;
     }
 
     StorePath build(ref<Store> store)
@@ -210,7 +210,7 @@ struct ProfileManifest
 
         buildProfile(tempDir, std::move(pkgs));
 
-        writeFile(tempDir + "/manifest.json", toJSON(*store));
+        writeFile(tempDir + "/manifest.json", toJSON(*store).dump());
 
         /* Add the symlink tree to the store. */
         StringSink sink;
@@ -635,7 +635,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
     }
 };
 
-struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
+struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON
 {
     std::string description() override
     {
@@ -653,18 +653,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
     {
         ProfileManifest manifest(*getEvalState(), *profile);
 
-        for (size_t i = 0; i < manifest.elements.size(); ++i) {
-            auto & element(manifest.elements[i]);
-            if (i) logger->cout("");
-            logger->cout("Index:              " ANSI_BOLD "%s" ANSI_NORMAL "%s",
-                i,
-                element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
-            if (element.source) {
-                logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
-                logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
-                logger->cout("Locked flake URL:   %s", element.source->lockedRef.to_string());
+        if (json) {
+            std::cout << manifest.toJSON(*store).dump() << "\n";
+        } else {
+            for (size_t i = 0; i < manifest.elements.size(); ++i) {
+                auto & element(manifest.elements[i]);
+                if (i) logger->cout("");
+                logger->cout("Index:              " ANSI_BOLD "%s" ANSI_NORMAL "%s",
+                    i,
+                    element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
+                if (element.source) {
+                    logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
+                    logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
+                    logger->cout("Locked flake URL:   %s", element.source->lockedRef.to_string());
+                }
+                logger->cout("Store paths:        %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
             }
-            logger->cout("Store paths:        %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
         }
     }
 };

From 5fbfbb4c7c246090df3debf7688f5ee9f01a4cbb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 May 2022 13:53:56 +0200
Subject: [PATCH 173/183] Fix test

---
 tests/nix-profile.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh
index 9da3f802b..7c478a0cd 100644
--- a/tests/nix-profile.sh
+++ b/tests/nix-profile.sh
@@ -47,8 +47,9 @@ cp ./config.nix $flake1Dir/
 
 # Test upgrading from nix-env.
 nix-env -f ./user-envs.nix -i foo-1.0
-nix profile list | grep '0 - - .*-foo-1.0'
+nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0'
 nix profile install $flake1Dir -L
+nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash'
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])

From 82d6699976aa51cb6c06272d3455b32299b2c4ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Tue, 4 Jul 2023 10:39:08 +0200
Subject: [PATCH 174/183] Document the path flakeref format (#8640)

* Document the path flakeref format

Fix https://github.com/NixOS/nix/issues/8482

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/nix/flake.md | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/nix/flake.md b/src/nix/flake.md
index 456fd0ea1..92f477917 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -71,8 +71,6 @@ inputs.nixpkgs = {
 
 Here are some examples of flake references in their URL-like representation:
 
-* `.`: The flake in the current directory.
-* `/home/alice/src/patchelf`: A flake in some other directory.
 * `nixpkgs`: The `nixpkgs` entry in the flake registry.
 * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs`
   entry in the flake registry, with its Git revision overridden to a
@@ -93,6 +91,23 @@ Here are some examples of flake references in their URL-like representation:
 * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball
   flake.
 
+## Path-like syntax
+
+Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity).
+
+The semantic of such a path is as follows:
+
+* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url;
+* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of:
+    1. The Git repository root, or
+    2. The filesystem root (/), or
+    3. A folder on a different mount point.
+
+### Examples
+
+* `.`: The flake to which the current directory belongs to.
+* `/home/alice/src/patchelf`: A flake in some other directory.
+
 ## Flake reference attributes
 
 The following generic flake reference attributes are supported:

From 4b2f155f0a22179e46d2c55cb1bd074c885fd08e Mon Sep 17 00:00:00 2001
From: Ana Hobden <operator@hoverbear.org>
Date: Thu, 6 Jul 2023 09:00:49 -0700
Subject: [PATCH 175/183] nix-daemon.service: Add TasksMax=infinity

---
 misc/systemd/nix-daemon.service.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in
index f46413630..4b4474475 100644
--- a/misc/systemd/nix-daemon.service.in
+++ b/misc/systemd/nix-daemon.service.in
@@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
 LimitNOFILE=1048576
+TasksMax=infinity
 
 [Install]
 WantedBy=multi-user.target

From 537e8beb770ed92dd1d5a20796a86620c4c61389 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Fri, 7 Jul 2023 11:00:40 +0200
Subject: [PATCH 176/183] fetchClosure: Apply suggestions from code review

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libexpr/primops/fetchClosure.cc | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 5114df553..22bae19b7 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -74,7 +74,10 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos
         throw Error({
             .msg = hintfmt(
                 "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n"
-                "If you do intend to fetch an input-addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.",
+                "If you do intend to fetch an input-addressed store path, add\n\n"
+                "    inputAddressed = true;\n\n"
+                "to the 'fetchClosure' arguments.\n\n"
+                "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.",
                 state.store->printStorePath(fromPath)),
             .errPos = state.positions[pos]
         });
@@ -226,7 +229,7 @@ static RegisterPrimOp primop_fetchClosure({
 
       - Fetch a content-addressed store path.
 
-Example:
+      Example:
 
         ```nix
         builtins.fetchClosure {
@@ -263,7 +266,7 @@ Example:
       rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
       ```
 
-      Alternatively, you may set `toPath = ""` and find the correct `toPath` in the error message.
+      Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
 
       **Not rewriting**
 
@@ -279,13 +282,7 @@ Example:
       allows you to use a previously built store path in a Nix
       expression. However, it is more reproducible because it requires
       specifying a binary cache from which the path can be fetched.
-      Also, the default requirement of a content-addressed final store path
-      avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys).
-
-      **Experimental feature**
-
-      This function is only available if you enable the experimental
-      feature `fetch-closure`.
+      Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
     )",
     .fun = prim_fetchClosure,
     .experimentalFeature = Xp::FetchClosure,

From b4b02d084f066d6391074ac807e532e36e4eccd9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 7 Jul 2023 11:40:40 +0200
Subject: [PATCH 177/183] fetchClosure: Interleave the examples in the docs

---
 src/libexpr/primops/fetchClosure.cc | 79 ++++++++++++++---------------
 1 file changed, 37 insertions(+), 42 deletions(-)

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 22bae19b7..7fe8203f4 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -211,52 +211,42 @@ static RegisterPrimOp primop_fetchClosure({
     .name = "__fetchClosure",
     .args = {"args"},
     .doc = R"(
-      Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache.
+      Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context.
 
-      This function can be used in three ways:
+      This function can be invoked in three ways, that we will discuss in order of preference.
 
-      - Fetch any store path and rewrite it to a fully content-addressed store path.
-
-        Example:
-
-        ```nix
-        builtins.fetchClosure {
-            fromStore = "https://cache.nixos.org";
-            fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
-            toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
-        }
-        ```
-
-      - Fetch a content-addressed store path.
+      **Fetch a content-addressed store path**
 
       Example:
 
-        ```nix
-        builtins.fetchClosure {
-            fromStore = "https://cache.nixos.org";
-            fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
-        }
-        ```
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+      }
+      ```
 
-      - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable.
+      This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
 
-        Example:
+      If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations.
 
-        ```nix
-        builtins.fetchClosure {
-            fromStore = "https://cache.nixos.org";
-            fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
-            inputAddressed = true;
-        }
-        ```
+      **Fetch any store path and rewrite it to a fully content-addressed store path**
 
-      **Rewriting a store path**
+      Example:
 
-      This first example fetches `/nix/store/r2jd...` from the specified binary cache,
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+        toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+      }
+      ```
+
+      This example fetches `/nix/store/r2jd...` from the specified binary cache,
       and rewrites it into the content-addressed store path
       `/nix/store/ldbh...`.
 
-      By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust.
+      Like the previous example, no extra configuration or privileges are required.
 
       To find out the correct value for `toPath` given a `fromPath`,
       use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
@@ -268,20 +258,25 @@ static RegisterPrimOp primop_fetchClosure({
 
       Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
 
-      **Not rewriting**
+      **Fetch an input-addressed store path as is**
 
-      `toPath` may be omitted when either
-       - `fromPath` is already content-addressed, or
-       - `inputAddressed = true;` is set.
+      Example:
 
-      Fetching an [input addressed path](@docroot@/glossary.md#gloss-input-addressed-store-object) is not recommended because it requires that the source be trusted by the Nix configuration.
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+        inputAddressed = true;
+      }
+      ```
+
+      It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is.
+      However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration.
 
       **`builtins.storePath`**
 
-      This function is similar to [`builtins.storePath`](#builtins-storePath) in that it
-      allows you to use a previously built store path in a Nix
-      expression. However, it is more reproducible because it requires
-      specifying a binary cache from which the path can be fetched.
+      `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression.
+      However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched.
       Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
     )",
     .fun = prim_fetchClosure,

From 9fc82de49388a58240234d10893673566432f7ab Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 7 Jul 2023 15:07:30 +0200
Subject: [PATCH 178/183] signing.sh: Revert test improvement because it fails
 on GHA + macOS

---
 tests/signing.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/signing.sh b/tests/signing.sh
index f14b53e7f..942b51630 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -84,7 +84,11 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2)
 # Copying to a diverted store should fail due to a lack of signatures by trusted keys.
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0
-expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key'
+
+# Fails or very flaky only on GHA + macOS:
+#     expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key'
+# but this works:
+(! nix copy --to $TEST_ROOT/store0 $outPath)
 
 # But succeed if we supply the public keys.
 nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1

From d76bf29c5f55f63f10741290c8d415c880a80ac2 Mon Sep 17 00:00:00 2001
From: Ana Hobden <operator@hoverbear.org>
Date: Fri, 7 Jul 2023 07:52:16 -0700
Subject: [PATCH 179/183] Choose a reasonable number similar to LimitNOFile

---
 misc/systemd/nix-daemon.service.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in
index 4b4474475..45fbea02c 100644
--- a/misc/systemd/nix-daemon.service.in
+++ b/misc/systemd/nix-daemon.service.in
@@ -10,7 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
 LimitNOFILE=1048576
-TasksMax=infinity
+TasksMax=1048576
 
 [Install]
 WantedBy=multi-user.target

From 87dcd090470ed6e56a2744cbe1490d2cf235d5c0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 23 Jun 2023 12:31:09 -0400
Subject: [PATCH 180/183] Clean up `resolveSearchPathElem`

We should use `std::optional<std::string>` not `std::pair<bool,
std::string>` for an optional string.
---
 src/libexpr/eval.cc  | 14 +++++++-------
 src/libexpr/eval.hh  |  8 ++++++--
 src/libexpr/parser.y | 45 +++++++++++++++++++++++---------------------
 3 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 706a19024..97a264085 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -571,22 +571,22 @@ EvalState::EvalState(
         allowedPaths = PathSet();
 
         for (auto & i : searchPath) {
-            auto r = resolveSearchPathElem(i);
-            if (!r.first) continue;
+            auto r = resolveSearchPathElem(i.path);
+            if (!r) continue;
 
-            auto path = r.second;
+            auto path = *std::move(r);
 
-            if (store->isInStore(r.second)) {
+            if (store->isInStore(path)) {
                 try {
                     StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                    store->computeFSClosure(store->toStorePath(path).first, closure);
                     for (auto & path : closure)
                         allowPath(path);
                 } catch (InvalidPath &) {
-                    allowPath(r.second);
+                    allowPath(path);
                 }
             } else
-                allowPath(r.second);
+                allowPath(path);
         }
     }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index e3676c1b7..e1a540a7f 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -317,7 +317,7 @@ private:
 
     SearchPath searchPath;
 
-    std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
+    std::map<std::string, std::optional<std::string>> searchPathResolved;
 
     /**
      * Cache used by checkSourcePath().
@@ -434,9 +434,13 @@ public:
     SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
+     * Try to resolve a search path value (not the optinal key part)
+     *
      * If the specified search path element is a URI, download it.
+     *
+     * If it is not found, return `std::nullopt`
      */
-    std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
+    std::optional<std::string> resolveSearchPathElem(const std::string & value);
 
     /**
      * Evaluate an expression to normal form
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 0d0004f9f..f839e804c 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -772,9 +772,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
                 continue;
             suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
         }
-        auto r = resolveSearchPathElem(i);
-        if (!r.first) continue;
-        Path res = r.second + suffix;
+        auto r = resolveSearchPathElem(i.path);
+        if (!r) continue;
+        Path res = *r + suffix;
         if (pathExists(res)) return CanonPath(canonPath(res));
     }
 
@@ -791,49 +791,52 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
 }
 
 
-std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem)
+std::optional<std::string> EvalState::resolveSearchPathElem(const std::string & value)
 {
-    auto i = searchPathResolved.find(elem.path);
+    auto i = searchPathResolved.find(value);
     if (i != searchPathResolved.end()) return i->second;
 
-    std::pair<bool, std::string> res;
+    std::optional<std::string> res;
 
-    if (EvalSettings::isPseudoUrl(elem.path)) {
+    if (EvalSettings::isPseudoUrl(value)) {
         try {
             auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath;
-            res = { true, store->toRealPath(storePath) };
+                store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
+            res = { store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path)
+                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
             });
-            res = { false, "" };
+            res = std::nullopt;
         }
     }
 
-    else if (hasPrefix(elem.path, "flake:")) {
+    else if (hasPrefix(value, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false);
-        debug("fetching flake search path element '%s''", elem.path);
+        auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", value);
         auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
-        res = { true, store->toRealPath(storePath) };
+        res = { store->toRealPath(storePath) };
     }
 
     else {
-        auto path = absPath(elem.path);
+        auto path = absPath(value);
         if (pathExists(path))
-            res = { true, path };
+            res = { path };
         else {
             logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
             });
-            res = { false, "" };
+            res = std::nullopt;
         }
     }
 
-    debug("resolved search path element '%s' to '%s'", elem.path, res.second);
+    if (res)
+        debug("resolved search path element '%s' to '%s'", value, *res);
+    else
+        debug("failed to resolve search path element '%s'", value);
 
-    searchPathResolved[elem.path] = res;
+    searchPathResolved[value] = res;
     return res;
 }
 

From be518e73ae331ac2f46e6b3a0ffdfeead26e3186 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 23 Jun 2023 13:51:25 -0400
Subject: [PATCH 181/183] Clean up `SearchPath`

- Better types

- Own header / C++ file pair

- Test factored out methods

- Pass parsed thing around more than strings

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libcmd/common-eval-args.cc   |   4 +-
 src/libcmd/common-eval-args.hh   |   3 +-
 src/libcmd/repl.cc               |   8 +--
 src/libcmd/repl.hh               |   2 +-
 src/libexpr/eval.cc              |  12 ++--
 src/libexpr/eval.hh              |  18 ++----
 src/libexpr/parser.y             |  47 +++++---------
 src/libexpr/primops.cc           |  14 ++--
 src/libexpr/search-path.cc       |  56 ++++++++++++++++
 src/libexpr/search-path.hh       | 108 +++++++++++++++++++++++++++++++
 src/libexpr/tests/search-path.cc |  90 ++++++++++++++++++++++++++
 src/nix/upgrade-nix.cc           |   2 +-
 12 files changed, 300 insertions(+), 64 deletions(-)
 create mode 100644 src/libexpr/search-path.cc
 create mode 100644 src/libexpr/search-path.hh
 create mode 100644 src/libexpr/tests/search-path.cc

diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index 7f97364a1..3df2c71a5 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -105,7 +105,9 @@ MixEvalArgs::MixEvalArgs()
   )",
         .category = category,
         .labels = {"path"},
-        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+        .handler = {[&](std::string s) {
+            searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
+        }}
     });
 
     addFlag({
diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh
index b65cb5b20..6359b2579 100644
--- a/src/libcmd/common-eval-args.hh
+++ b/src/libcmd/common-eval-args.hh
@@ -3,6 +3,7 @@
 
 #include "args.hh"
 #include "common-args.hh"
+#include "search-path.hh"
 
 namespace nix {
 
@@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
 
     Bindings * getAutoArgs(EvalState & state);
 
-    Strings searchPath;
+    SearchPath searchPath;
 
     std::optional<std::string> evalStoreUrl;
 
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 4b160a100..f9e9c2bf8 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -68,7 +68,7 @@ struct NixRepl
 
     const Path historyFile;
 
-    NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state,
+    NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
             std::function<AnnotatedValues()> getValues);
     virtual ~NixRepl();
 
@@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s)
 }
 
 
-NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
             std::function<NixRepl::AnnotatedValues()> getValues)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
@@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
-   const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+   const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
    std::function<AnnotatedValues()> getValues)
 {
     return std::make_unique<NixRepl>(
@@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple(
         NixRepl::AnnotatedValues values;
         return values;
     };
-    const Strings & searchPath = {};
+    SearchPath searchPath = {};
     auto repl = std::make_unique<NixRepl>(
             searchPath,
             openStore(),
diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh
index 731c8e6db..6d88883fe 100644
--- a/src/libcmd/repl.hh
+++ b/src/libcmd/repl.hh
@@ -25,7 +25,7 @@ struct AbstractNixRepl
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
     static std::unique_ptr<AbstractNixRepl> create(
-        const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+        const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
         std::function<AnnotatedValues()> getValues);
 
     static void runSimple(
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 97a264085..be1bdb806 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -498,7 +498,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr)
 
 
 EvalState::EvalState(
-    const Strings & _searchPath,
+    const SearchPath & _searchPath,
     ref<Store> store,
     std::shared_ptr<Store> buildStore)
     : sWith(symbols.create("<with>"))
@@ -563,15 +563,17 @@ EvalState::EvalState(
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        for (auto & i : _searchPath) addToSearchPath(i);
-        for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
+        for (auto & i : _searchPath.elements)
+            addToSearchPath(SearchPath::Elem {i});
+        for (auto & i : evalSettings.nixPath.get())
+            addToSearchPath(SearchPath::Elem::parse(i));
     }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
 
-        for (auto & i : searchPath) {
-            auto r = resolveSearchPathElem(i.path);
+        for (auto & i : searchPath.elements) {
+            auto r = resolveSearchPathPath(i.path);
             if (!r) continue;
 
             auto path = *std::move(r);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index e1a540a7f..277e77ad5 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -9,6 +9,7 @@
 #include "config.hh"
 #include "experimental-features.hh"
 #include "input-accessor.hh"
+#include "search-path.hh"
 
 #include <map>
 #include <optional>
@@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
-struct SearchPathElem
-{
-    std::string prefix;
-    // FIXME: maybe change this to an std::variant<SourcePath, URL>.
-    std::string path;
-};
-typedef std::list<SearchPathElem> SearchPath;
-
-
 /**
  * Initialise the Boehm GC, if applicable.
  */
@@ -344,12 +336,12 @@ private:
 public:
 
     EvalState(
-        const Strings & _searchPath,
+        const SearchPath & _searchPath,
         ref<Store> store,
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void addToSearchPath(const std::string & s);
+    void addToSearchPath(SearchPath::Elem && elem);
 
     SearchPath getSearchPath() { return searchPath; }
 
@@ -431,7 +423,7 @@ public:
      * Look up a file in the search path.
      */
     SourcePath findFile(const std::string_view path);
-    SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
+    SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
      * Try to resolve a search path value (not the optinal key part)
@@ -440,7 +432,7 @@ public:
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveSearchPathElem(const std::string & value);
+    std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
 
     /**
      * Evaluate an expression to normal form
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index f839e804c..77bcc31e0 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -734,22 +734,9 @@ Expr * EvalState::parseStdin()
 }
 
 
-void EvalState::addToSearchPath(const std::string & s)
+void EvalState::addToSearchPath(SearchPath::Elem && elem)
 {
-    size_t pos = s.find('=');
-    std::string prefix;
-    Path path;
-    if (pos == std::string::npos) {
-        path = s;
-    } else {
-        prefix = std::string(s, 0, pos);
-        path = std::string(s, pos + 1);
-    }
-
-    searchPath.emplace_back(SearchPathElem {
-        .prefix = prefix,
-        .path = path,
-    });
+    searchPath.elements.emplace_back(std::move(elem));
 }
 
 
@@ -759,22 +746,19 @@ SourcePath EvalState::findFile(const std::string_view path)
 }
 
 
-SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
 {
-    for (auto & i : searchPath) {
-        std::string suffix;
-        if (i.prefix.empty())
-            suffix = concatStrings("/", path);
-        else {
-            auto s = i.prefix.size();
-            if (path.compare(0, s, i.prefix) != 0 ||
-                (path.size() > s && path[s] != '/'))
-                continue;
-            suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
-        }
-        auto r = resolveSearchPathElem(i.path);
-        if (!r) continue;
-        Path res = *r + suffix;
+    for (auto & i : searchPath.elements) {
+        auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
+
+        if (!suffixOpt) continue;
+        auto suffix = *suffixOpt;
+
+        auto rOpt = resolveSearchPathPath(i.path);
+        if (!rOpt) continue;
+        auto r = *rOpt;
+
+        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
         if (pathExists(res)) return CanonPath(canonPath(res));
     }
 
@@ -791,8 +775,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
 }
 
 
-std::optional<std::string> EvalState::resolveSearchPathElem(const std::string & value)
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
 {
+    auto & value = value0.s;
     auto i = searchPathResolved.find(value);
     if (i != searchPathResolved.end()) return i->second;
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5dfad470a..b98b06db9 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1656,9 +1656,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
             }));
         }
 
-        searchPath.emplace_back(SearchPathElem {
-            .prefix = prefix,
-            .path = path,
+        searchPath.elements.emplace_back(SearchPath::Elem {
+            .prefix = SearchPath::Prefix { .s = prefix },
+            .path = SearchPath::Path { .s = path },
         });
     }
 
@@ -4319,12 +4319,12 @@ void EvalState::createBaseEnv()
     });
 
     /* Add a value containing the current Nix expression search path. */
-    mkList(v, searchPath.size());
+    mkList(v, searchPath.elements.size());
     int n = 0;
-    for (auto & i : searchPath) {
+    for (auto & i : searchPath.elements) {
         auto attrs = buildBindings(2);
-        attrs.alloc("path").mkString(i.path);
-        attrs.alloc("prefix").mkString(i.prefix);
+        attrs.alloc("path").mkString(i.path.s);
+        attrs.alloc("prefix").mkString(i.prefix.s);
         (v.listElems()[n++] = allocValue())->mkAttrs(attrs);
     }
     addConstant("__nixPath", v, {
diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc
new file mode 100644
index 000000000..36bb4c3a5
--- /dev/null
+++ b/src/libexpr/search-path.cc
@@ -0,0 +1,56 @@
+#include "search-path.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
+    std::string_view path) const
+{
+    auto n = s.size();
+
+    /* Non-empty prefix and suffix must be separated by a /, or the
+       prefix is not a valid path prefix. */
+    bool needSeparator = n > 0 && (path.size() - n) > 0;
+
+    if (needSeparator && path[n] != '/') {
+        return std::nullopt;
+    }
+
+    /* Prefix must be prefix of this path. */
+    if (path.compare(0, n, s) != 0) {
+        return std::nullopt;
+    }
+
+    /* Skip next path separator. */
+    return {
+        path.substr(needSeparator ? n + 1 : n)
+    };
+}
+
+
+SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
+{
+    size_t pos = rawElem.find('=');
+
+    return SearchPath::Elem {
+        .prefix = Prefix {
+            .s = pos == std::string::npos
+                ? std::string { "" }
+                : std::string { rawElem.substr(0, pos) },
+        },
+        .path = Path {
+            .s = std::string { rawElem.substr(pos + 1) },
+        },
+    };
+}
+
+
+SearchPath parseSearchPath(const Strings & rawElems)
+{
+    SearchPath res;
+    for (auto & rawElem : rawElems)
+        res.elements.emplace_back(SearchPath::Elem::parse(rawElem));
+    return res;
+}
+
+}
diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh
new file mode 100644
index 000000000..ce78135b5
--- /dev/null
+++ b/src/libexpr/search-path.hh
@@ -0,0 +1,108 @@
+#pragma once
+///@file
+
+#include <optional>
+
+#include "types.hh"
+#include "comparator.hh"
+
+namespace nix {
+
+/**
+ * A "search path" is a list of ways look for something, used with
+ * `builtins.findFile` and `< >` lookup expressions.
+ */
+struct SearchPath
+{
+    /**
+     * A single element of a `SearchPath`.
+     *
+     * Each element is tried in succession when looking up a path. The first
+     * element to completely match wins.
+     */
+    struct Elem;
+
+    /**
+     * The first part of a `SearchPath::Elem` pair.
+     *
+     * Called a "prefix" because it takes the form of a prefix of a file
+     * path (first `n` path components). When looking up a path, to use
+     * a `SearchPath::Elem`, its `Prefix` must match the path.
+     */
+    struct Prefix;
+
+    /**
+     * The second part of a `SearchPath::Elem` pair.
+     *
+     * It is either a path or a URL (with certain restrictions / extra
+     * structure).
+     *
+     * If the prefix of the path we are looking up matches, we then
+     * check if the rest of the path points to something that exists
+     * within the directory denoted by this. If so, the
+     * `SearchPath::Elem` as a whole matches, and that *something* being
+     * pointed to by the rest of the path we are looking up is the
+     * result.
+     */
+    struct Path;
+
+    /**
+     * The list of search path elements. Each one is checked for a path
+     * when looking up. (The actual lookup entry point is in `EvalState`
+     * not in this class.)
+     */
+    std::list<SearchPath::Elem> elements;
+
+    /**
+     * Parse a string into a `SearchPath`
+     */
+    static SearchPath parse(const Strings & rawElems);
+};
+
+struct SearchPath::Prefix
+{
+    /**
+     * Underlying string
+     *
+     * @todo Should we normalize this when constructing a `SearchPath::Prefix`?
+     */
+    std::string s;
+
+    GENERATE_CMP(SearchPath::Prefix, me->s);
+
+    /**
+     * If the path possibly matches this search path element, return the
+     * suffix that we should look for inside the resolved value of the
+     * element
+     * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist.
+     */
+    std::optional<std::string_view> suffixIfPotentialMatch(std::string_view path) const;
+};
+
+struct SearchPath::Path
+{
+    /**
+     * The location of a search path item, as a path or URL.
+     *
+     * @todo Maybe change this to `std::variant<SourcePath, URL>`.
+     */
+    std::string s;
+
+    GENERATE_CMP(SearchPath::Path, me->s);
+};
+
+struct SearchPath::Elem
+{
+
+    Prefix prefix;
+    Path path;
+
+    GENERATE_CMP(SearchPath::Elem, me->prefix, me->path);
+
+    /**
+     * Parse a string into a `SearchPath::Elem`
+     */
+    static SearchPath::Elem parse(std::string_view rawElem);
+};
+
+}
diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc
new file mode 100644
index 000000000..dbe7ab95f
--- /dev/null
+++ b/src/libexpr/tests/search-path.cc
@@ -0,0 +1,90 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "search-path.hh"
+
+namespace nix {
+
+TEST(SearchPathElem, parse_justPath) {
+    ASSERT_EQ(
+        SearchPath::Elem::parse("foo"),
+        (SearchPath::Elem {
+            .prefix = SearchPath::Prefix { .s = "" },
+            .path = SearchPath::Path { .s = "foo" },
+        }));
+}
+
+TEST(SearchPathElem, parse_emptyPrefix) {
+    ASSERT_EQ(
+        SearchPath::Elem::parse("=foo"),
+        (SearchPath::Elem {
+            .prefix = SearchPath::Prefix { .s = "" },
+            .path = SearchPath::Path { .s = "foo" },
+        }));
+}
+
+TEST(SearchPathElem, parse_oneEq) {
+    ASSERT_EQ(
+        SearchPath::Elem::parse("foo=bar"),
+        (SearchPath::Elem {
+            .prefix = SearchPath::Prefix { .s = "foo" },
+            .path = SearchPath::Path { .s = "bar" },
+        }));
+}
+
+TEST(SearchPathElem, parse_twoEqs) {
+    ASSERT_EQ(
+        SearchPath::Elem::parse("foo=bar=baz"),
+        (SearchPath::Elem {
+            .prefix = SearchPath::Prefix { .s = "foo" },
+            .path = SearchPath::Path { .s = "bar=baz" },
+        }));
+}
+
+
+TEST(SearchPathElem, suffixIfPotentialMatch_justPath) {
+    SearchPath::Prefix prefix { .s = "" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" });
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt);
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt);
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) {
+    SearchPath::Prefix prefix { .s = "fooX" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt);
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" });
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) {
+    SearchPath::Prefix prefix { .s = "foo/bar" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" });
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" });
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" });
+}
+
+TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) {
+    SearchPath::Prefix prefix { .s = "foo" };
+    ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" });
+}
+
+}
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 3997c98bf..d05c23fb7 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         auto req = FileTransferRequest(storePathsUrl);
         auto res = getFileTransfer()->download(req);
 
-        auto state = std::make_unique<EvalState>(Strings(), store);
+        auto state = std::make_unique<EvalState>(SearchPath{}, store);
         auto v = state->allocValue();
         state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v);
         Bindings & bindings(*state->allocBindings(0));

From 3d74e7b811ea8fc13e31127175be12ba8d39351c Mon Sep 17 00:00:00 2001
From: Yingchi Long <i@lyc.dev>
Date: Mon, 10 Jul 2023 12:02:29 +0800
Subject: [PATCH 182/183] libexpr: remove std::move() for `basePath` in parser,
 it has no effect

---
 src/libexpr/parser.y | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 0d0004f9f..3336d3315 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -663,7 +663,7 @@ Expr * EvalState::parse(
     ParseData data {
         .state = *this,
         .symbols = symbols,
-        .basePath = std::move(basePath),
+        .basePath = basePath,
         .origin = {origin},
     };
 

From 3fa0266e7a736e85e2faaa6c92ab23e809f89930 Mon Sep 17 00:00:00 2001
From: Bader AlAttar <68021685+balattar@users.noreply.github.com>
Date: Mon, 10 Jul 2023 02:33:04 -0700
Subject: [PATCH 183/183] Fix some grammar in installables doc (#8682)

---
 src/nix/nix.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/nix.md b/src/nix/nix.md
index 6d9e40dbc..e0f459d6b 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -63,7 +63,7 @@ The following types of installable are supported by most commands:
 - [Nix file](#nix-file), optionally qualified by an attribute path
 - [Nix expression](#nix-expression), optionally qualified by an attribute path
 
-For most commands, if no installable is specified, `.` as assumed.
+For most commands, if no installable is specified, `.` is assumed.
 That is, Nix will operate on the default flake output attribute of the flake in the current directory.
 
 ### Flake output attribute